forked from dagster-io/dagster
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathinstall_dev_python_modules.py
163 lines (145 loc) · 7.08 KB
/
install_dev_python_modules.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
# ruff: noqa: T201
import argparse
import subprocess
import sys
from typing import List, Optional
# We allow extra packages to be passed in via the command line because pip's version resolution
# requires everything to be installed at the same time.
parser = argparse.ArgumentParser()
parser.add_argument("-q", "--quiet", action="count")
parser.add_argument(
"packages",
type=str,
nargs="*",
help="Additional packages (with optional version reqs) to pass to `pip install`",
)
parser.add_argument("--include-prebuilt-grpcio-wheel", action="store_true")
parser.add_argument(
"--system",
action="store_true",
help="Install the packages into the system Python. Should only be used in Dockferfiles or CI/CD.",
)
def main(
quiet: bool,
extra_packages: List[str],
include_prebuilt_grpcio_wheel: Optional[bool],
system: Optional[bool],
) -> None:
"""Especially on macOS, there may be missing wheels for new major Python versions, which means that
some dependencies may have to be built from source. You may find yourself needing to install
system packages such as freetype, gfortran, etc.; on macOS, Homebrew should suffice.
"""
# Previously, we did a pip install --upgrade pip here. We have removed that and instead
# depend on the user to ensure an up-to-date pip is installed and available. If you run into
# build errors, try this first. For context, there is a lengthy discussion here:
# https://github.com/pypa/pip/issues/5599
install_targets: List[str] = [
*extra_packages,
]
# Not all libs are supported on all Python versions. Consult `dagster_buildkite.steps.packages`
# as the source of truth on which packages support which Python versions. The building of
# `install_targets` below should use `sys.version_info` checks to reflect this.
# Supported on all Python versions.
install_targets += [
"-e python_modules/dagster[pyright,ruff,test]",
"-e python_modules/dagster-pipes",
"-e python_modules/dagster-graphql",
"-e python_modules/dagster-test",
"-e python_modules/dagster-webserver",
"-e python_modules/dagit",
"-e python_modules/automation",
"-e python_modules/libraries/dagster-managed-elements",
"-e python_modules/libraries/dagster-airbyte",
"-e python_modules/libraries/dagster-aws[test]",
"-e python_modules/libraries/dagster-celery",
"-e python_modules/libraries/dagster-celery-docker",
"-e python_modules/libraries/dagster-dask[yarn,pbs,kube]",
"-e python_modules/libraries/dagster-databricks",
"-e python_modules/libraries/dagster-datadog",
"-e python_modules/libraries/dagster-datahub",
"-e python_modules/libraries/dagster-dbt",
"-e python_modules/libraries/dagster-docker",
"-e python_modules/libraries/dagster-gcp",
"-e python_modules/libraries/dagster-gcp-pandas",
"-e python_modules/libraries/dagster-gcp-pyspark",
"-e python_modules/libraries/dagster-embedded-elt",
"-e python_modules/libraries/dagster-fivetran",
"-e python_modules/libraries/dagster-k8s",
"-e python_modules/libraries/dagster-celery-k8s",
"-e python_modules/libraries/dagster-github",
"-e python_modules/libraries/dagster-mlflow",
"-e python_modules/libraries/dagster-mysql",
"-e python_modules/libraries/dagster-looker",
"-e python_modules/libraries/dagster-pagerduty",
"-e python_modules/libraries/dagster-pandas",
"-e python_modules/libraries/dagster-papertrail",
"-e python_modules/libraries/dagster-postgres",
"-e python_modules/libraries/dagster-prometheus",
"-e python_modules/libraries/dagster-pyspark",
"-e python_modules/libraries/dagster-shell",
"-e python_modules/libraries/dagster-slack",
"-e python_modules/libraries/dagster-spark",
"-e python_modules/libraries/dagster-ssh",
"-e python_modules/libraries/dagster-twilio",
"-e python_modules/libraries/dagstermill",
"-e integration_tests/python_modules/dagster-k8s-test-infra",
"-e python_modules/libraries/dagster-azure",
"-e python_modules/libraries/dagster-msteams",
"-e python_modules/libraries/dagster-deltalake",
"-e python_modules/libraries/dagster-deltalake-pandas",
"-e python_modules/libraries/dagster-deltalake-polars",
"-e helm/dagster/schema[test]",
"-e .buildkite/dagster-buildkite",
"-e examples/experimental/dagster-blueprints",
]
if sys.version_info <= (3, 12):
install_targets += [
"-e python_modules/libraries/dagster-duckdb",
"-e python_modules/libraries/dagster-duckdb-pandas",
"-e python_modules/libraries/dagster-duckdb-polars",
"-e python_modules/libraries/dagster-duckdb-pyspark",
"-e python_modules/libraries/dagster-wandb",
"-e python_modules/libraries/dagster-airflow",
]
if sys.version_info > (3, 7):
install_targets += [
"-e python_modules/libraries/dagster-pandera",
"-e python_modules/libraries/dagster-snowflake",
"-e python_modules/libraries/dagster-snowflake-pandas",
"-e python_modules/libraries/dagster-polars[deltalake,gcp,test]",
]
if sys.version_info > (3, 6) and sys.version_info < (3, 10):
install_targets += []
if include_prebuilt_grpcio_wheel:
install_targets += [
"--find-links",
"https://github.com/dagster-io/build-grpcio/wiki/Wheels",
]
# NOTE: `dagster-ge` is out of date and does not support recent versions of great expectations.
# Because of this, it has second-order dependencies on old versions of popular libraries like
# numpy which conflict with the requirements of our other libraries. For this reason, until
# dagster-ge is updated we won't install `dagster-ge` in the common dev environment or
# pre-install its dependencies in our BK images (which this script is used for).
#
# dagster-ge depends on a great_expectations version that does not install on Windows
# https://github.com/dagster-io/dagster/issues/3319
# if sys.version_info >= (3, 7) and os.name != "nt":
# install_targets += ["-e python_modules/libraries/dagster-ge"]
# Ensure uv is installed which we use for faster package resolution
subprocess.run(["pip", "install", "-U", "uv"], check=True)
# NOTE: These need to be installed as one long pip install command, otherwise pip will install
# conflicting dependencies, which will break pip freeze snapshot creation during the integration
# image build!
cmd = ["uv", "pip", "install"] + (["--system"] if system else []) + install_targets
if quiet is not None:
cmd.append(f'-{"q" * quiet}')
print(" ".join(cmd))
subprocess.run(cmd, check=True)
if __name__ == "__main__":
args = parser.parse_args()
main(
quiet=args.quiet,
extra_packages=args.packages,
include_prebuilt_grpcio_wheel=args.include_prebuilt_grpcio_wheel,
system=args.system,
)