[docs examples] move old examples to examples/legacy_examples

Summary: - update all the places where `pip install -e examples` to `pip install -e examples/legacy_examples`

Test Plan:
bk
`ls examples`
{F167292}

Reviewers: nate, sashank

Reviewed By: nate

Differential Revision: https://dagster.phacility.com/D3572

.buildkite/hooks/pre-exit

@@ -1,6 +1,6 @@
 set -eu
 
-pushd examples
+pushd examples/legacy_examples
 docker-compose stop
 docker-compose rm -f
 popd

.buildkite/pipeline.py

@@ -83,7 +83,7 @@ def airflow_extra_cmds_fn(version):
 
 def airline_demo_extra_cmds_fn(_):
     return [
-        'pushd examples',
+        'pushd examples/legacy_examples',
         # Build the image we use for airflow in the demo tests
         './build_airline_demo_image.sh',
         'mkdir -p /home/circleci/airflow',
@@ -130,9 +130,9 @@ def dask_extra_cmds_fn(version):
     ]
 
 
-def examples_extra_cmds_fn(_):
+def legacy_examples_extra_cmds_fn(_):
     return [
-        "pushd examples",
+        "pushd examples/legacy_examples",
         "docker-compose up -d --remove-orphans",  # clean up in hooks/pre-exit,
         # Can't use host networking on buildkite and communicate via localhost
         # between these sibling containers, so pass along the ip.
@@ -185,7 +185,7 @@ def postgres_extra_cmds_fn(_):
 DAGSTER_PACKAGES_WITH_CUSTOM_TESTS = [
     # Examples: Airline Demo
     ModuleBuildSpec(
-        'examples',
+        'examples/legacy_examples',
         supported_pythons=SupportedPython3sNo38,
         extra_cmds_fn=airline_demo_extra_cmds_fn,
         tox_file='tox_airline.ini',
@@ -202,15 +202,15 @@ def postgres_extra_cmds_fn(_):
     # ),
     # Examples
     ModuleBuildSpec(
-        'examples',
+        'examples/legacy_examples',
         # See: https://github.com/dagster-io/dagster/issues/1960
         # Also, examples are py3-only
         supported_pythons=SupportedPython3sNo38,
-        extra_cmds_fn=examples_extra_cmds_fn,
+        extra_cmds_fn=legacy_examples_extra_cmds_fn,
     ),
     ModuleBuildSpec(
         'examples/docs_snippets',
-        extra_cmds_fn=examples_extra_cmds_fn,
+        extra_cmds_fn=legacy_examples_extra_cmds_fn,
         upload_coverage=False,
         supported_pythons=SupportedPython3s,
     ),
@@ -354,10 +354,7 @@ def examples_tests():
     skip_examples = [
         # Skip these folders because they need custom build config
         'docs_snippets',
-        # Temporarily skip these folders until we finish the work to clean up the examples folder
-        'dagster_examples',
-        'dagster_examples_tests',
-        'data',
+        'legacy_examples',
     ]
 
     examples_root = os.path.join(SCRIPT_PATH, '..', 'examples')
@@ -447,7 +444,7 @@ def pylint_steps():
                 -e python_modules/libraries/dagstermill \
                 -e python_modules/libraries/dagster-celery \
                 -e python_modules/libraries/dagster-dask \
-                -e examples
+                -e examples/legacy_examples
             """,
             "pylint -j 0 `git ls-files %s` --rcfile=.pylintrc" % ' '.join(base_paths_ext),
         )
@@ -554,7 +551,7 @@ def version_equality_checks(version=SupportedPython.V3_7):
             "pip install -e python_modules/libraries/dagster-cron -qqq",
             "pip install -e python_modules/libraries/dagster-slack -qqq",
             "pip install -e python_modules/dagit -qqq",
-            "pip install -e examples -qqq",
+            "pip install -e examples/legacy_examples -qqq",
             "cd js_modules/dagit",
             "yarn install --offline",
             "yarn run ts",
@@ -573,8 +570,8 @@ def version_equality_checks(version=SupportedPython.V3_7):
         .run(
             "pip install mypy",
             # start small by making sure the local code type checks
-            "mypy examples/dagster_examples/airline_demo "
-            "examples/dagster_examples/bay_bikes "
+            "mypy examples/legacy_examples/dagster_examples/airline_demo "
+            "examples/legacy_examples/dagster_examples/bay_bikes "
             "examples/docs_snippets/docs_snippets/intro_tutorial/basics/e04_quality/custom_types_mypy* "
             "--ignore-missing-imports",
         )

.gitignore

@@ -123,6 +123,9 @@ docs/gatsby/**/public
 docs/next/.mdx-data
 # Data
 data
+# Don't ignore data folders in examples
+!examples/*/data
+!examples/**/**/data
 
 # Dask
 dask-worker-space
@@ -134,7 +137,7 @@ dask-worker-space
 .codemod.bookmark
 
 # Examples outputs
-examples/dagster_examples/*/output/
+examples/legacy_examples/dagster_examples/*/output/
 examples/docs_snippets/docs_snippets/**/**/output/
 
 # Telemetry instance id

Makefile

@@ -64,7 +64,7 @@ install_dev_python_modules:
 				-r python_modules/dagster/dev-requirements.txt \
 				-r python_modules/libraries/dagster-aws/dev-requirements.txt \
 				-r bin/requirements.txt \
-				-e examples[full] \
+				-e examples/legacy_examples[full] \
 				-r scala_modules/scripts/requirements.txt $(QUIET)
 
 	# Don't install dagster-azure as part of this target _yet_ - it has a dependency
@@ -99,4 +99,4 @@ dev_install: install_dev_python_modules_verbose rebuild_dagit
 dev_install_quiet: install_dev_python_modules rebuild_dagit
 
 graphql_tests:
-	pytest examples/dagster_examples_tests/graphql_tests/ python_modules/dagster-graphql/dagster_graphql_tests/graphql/ -s -vv
+	pytest examples/legacy_examples/dagster_examples_tests/graphql_tests/ python_modules/dagster-graphql/dagster_graphql_tests/graphql/ -s -vv

docs/next/src/pages/docs/learn/demos/airline_demo.mdx

@@ -46,7 +46,7 @@ pip install -e .[full]
 docker-compose up -d
 
 # Load the airline demo in Dagit
-cd dagster_examples/airline_demo
+cd legacy_examples/dagster_examples/airline_demo
 dagit
 ```
 
@@ -120,7 +120,7 @@ Let's start by looking at the pipeline definition (in
 to reuse the `s3_to_df` solid for several ingest steps:
 
 ```python literalinclude caption=pipelines.py
-file:/dagster_examples/airline_demo/pipelines.py
+file:/legacy_examples/dagster_examples/airline_demo/pipelines.py
 lines:96-103
 ```
 
@@ -158,7 +158,7 @@ to interact with the external environment. In YAML, an entry in the
 config for one of our solids aliasing `s3_to_df` looks like this:
 
 ```yaml literalinclude caption=local_fast_ingest.yaml
-file:/dagster_examples/airline_demo/environments/local_fast_ingest.yaml
+file:/legacy_examples/dagster_examples/airline_demo/environments/local_fast_ingest.yaml
 lines:32-38
 ```
 
@@ -224,7 +224,7 @@ CI/CD, with access to a production or replica environment, but where
 speed is of the essence; and remotely in production on live data.
 
 ```python literalinclude caption=hello_cereal.py
-file:/dagster_examples/airline_demo/pipelines.py
+file:/legacy_examples/dagster_examples/airline_demo/pipelines.py
 lines:46-69
 ```
 
@@ -270,7 +270,7 @@ authentication schema.
 Finally, we bring it all together in the `postgres_db_info_resource`:
 
 ```python literalinclude caption=resources.py
-file:/dagster_examples/common/resources.py
+file:/legacy_examples/dagster_examples/common/resources.py
 lines:89-128
 ```
 
@@ -306,7 +306,7 @@ The terminal nodes of this pipeline are all aliased instances of
 `load_data_to_database_from_spark`:
 
 ```python literalinclude caption=solids.py
-file:/dagster_examples/airline_demo/solids.py
+file:/legacy_examples/dagster_examples/airline_demo/solids.py
 lines:198-216
 ```
 
@@ -354,7 +354,7 @@ the data warehouse, an analyst only needs to supply light metadata along
 with their SQL query:
 
 ```python literalinclude caption=solids.py
-file:/dagster_examples/airline_demo/solids.py
+file:/legacy_examples/dagster_examples/airline_demo/solids.py
 lines:403-429
 ```
 
@@ -393,7 +393,7 @@ down on the tag:
 Let's start with the definition of our `notebook_solid` helper:
 
 ```python literalinclude caption=solids.py
-file:/dagster_examples/airline_demo/solids.py
+file:/legacy_examples/dagster_examples/airline_demo/solids.py
 lines:38-39
 ```
 
@@ -402,7 +402,7 @@ which tells Dagstermill where to look for the notebooks. We define a new
 solid by using this function and referencing a notebook file:
 
 ```python literalinclude caption=solids.py
-file:/dagster_examples/airline_demo/solids.py
+file:/legacy_examples/dagster_examples/airline_demo/solids.py
 lines:431-453
 ```
 

docs/next/src/pages/docs/learn/guides/dagster_pandas/dagster_pandas.mdx

@@ -24,15 +24,15 @@ list of `PandasColumn` objects which specify column-level schema and constraints
 can construct a custom dataframe type to represent a set of e-bike trips in the following way:
 
 ```python literalinclude caption=core_trip_pipeline.py
-file:/dagster_examples/dagster_pandas_guide/core_trip_pipeline.py
+file:/legacy_examples/dagster_examples/dagster_pandas_guide/core_trip_pipeline.py
 lines:9-22
 ```
 
 Once our custom data type is defined, we can use it as the type declaration for the inputs / outputs
 of our solid:
 
 ```python literalinclude caption=hello_cereal.py
-file:/dagster_examples/dagster_pandas_guide/core_trip_pipeline.py
+file:/legacy_examples/dagster_examples/dagster_pandas_guide/core_trip_pipeline.py
 lines:25-31
 ```
 
@@ -54,7 +54,7 @@ in the `dagster_pandas` [API docs](/docs/apidocs/libraries/dagster_pandas).
 This looks like:
 
 ```python literalinclude caption=shape_constrained_pipeline.py
-file:/dagster_examples/dagster_pandas_guide/shape_constrained_pipeline.py
+file:/legacy_examples/dagster_examples/dagster_pandas_guide/shape_constrained_pipeline.py
 lines:9-11
 ```
 
@@ -73,7 +73,7 @@ In complex pipelines, this can help debug and monitor data drift over
 time. Let's illustrate how this works in our example:
 
 ```python literalinclude caption=summary_stats_pipeline.py
-file:/dagster_examples/dagster_pandas_guide/summary_stats_pipeline.py
+file:/legacy_examples/dagster_examples/dagster_pandas_guide/summary_stats_pipeline.py
 lines:10-39
 ```
 
@@ -94,6 +94,6 @@ validation method which throws a `ColumnConstraintViolationException` if a row f
 This would look like the following:
 
 ```python literalinclude emphasize-lines=22-25 caption=custom_column_constraint_pipeline.py
-file:/dagster_examples/dagster_pandas_guide/custom_column_constraint_pipeline.py
+file:/legacy_examples/dagster_examples/dagster_pandas_guide/custom_column_constraint_pipeline.py
 lines:15-40
 ```

docs/next/src/pages/docs/learn/guides/partitioning/date_partition_set.mdx

@@ -14,15 +14,15 @@ create a <PyObject module="dagster" object="PartitionSetDefinition" /> for a set
 First, let's modify our solid to take a date range as config:
 
 ```python literalinclude caption=hello_cereal.py
-file:/dagster_examples/stocks/repository.py
+file:/legacy_examples/dagster_examples/stocks/repository.py
 lines:4-19
 ```
 
 Just as before, we write a function that returns a list of partitions,
 but this time we return a list of `datetime` objects.
 
 ```python literalinclude caption=date_partitions.py
-file:/dagster_examples/stocks/date_partitions.py
+file:/legacy_examples/dagster_examples/stocks/date_partitions.py
 lines:9-15
 ```
 
@@ -31,7 +31,7 @@ returns config. Here, we calculate the first and last days of the
 previous month for each date partition to pass to the solid config.
 
 ```python literalinclude caption=hello_cereal.py
-file:/dagster_examples/stocks/date_partitions.py
+file:/legacy_examples/dagster_examples/stocks/date_partitions.py
 lines:18-34
 ```
 
@@ -42,7 +42,7 @@ and time interval: <PyObject module="dagster.utils.partitions" object="date_part
 We'll use this function instead of the one we wrote above.
 
 ```python literalinclude caption=repository.py
-file:/dagster_examples/stocks/date_partitions.py
+file:/legacy_examples/dagster_examples/stocks/date_partitions.py
 lines:37-46
 ```
 

docs/next/src/pages/docs/learn/guides/partitioning/index.mdx

@@ -16,7 +16,7 @@ import { DynamicMetaTags } from 'components/MetaTags';
 - [Scheduling Date Partitions](/docs/learn/guides/partitioning/schedule_partitions)
 
 You can find all of the code for this guide checked into the dagster
-repository at `dagster/examples/dagster_examples/stocks`
+repository at `dagster/examples/legacy_examples/legacy_examples/legacy_examples/dagster_examples/stocks`
 
 In this guide, well learn about why we partition data and how to write
 Dagster pipelines that can easily work with partitioned data. Well

docs/next/src/pages/docs/learn/guides/partitioning/partition_pipeline.mdx

@@ -92,15 +92,15 @@ the API to return the volume traded from Jan 01 - Jan 31 2019. We pass
 the json response to the downstream solid.
 
 ```python literalinclude emphasize-lines=4-16 caption=repository.py
-file:/dagster_examples/stocks/pipelines_stock.py
+file:/legacy_examples/dagster_examples/stocks/pipelines_stock.py
 lines:5-20
 ```
 
 Our second solid will take the json response and total the volume
 amounts over all the days:
 
 ```python literalinclude caption=repository.py
-file:/dagster_examples/stocks/pipelines_stock.py
+file:/legacy_examples/dagster_examples/stocks/pipelines_stock.py
 lines:23-32
 ```
 

docs/next/src/pages/docs/learn/guides/partitioning/partition_set.mdx

@@ -22,7 +22,7 @@ First, we write a function that returns a list of partitions. Here, our
 partitions are simply the stock tickers, and we wrap them in <PyObject module="dagster" object="Partition"/> objects.
 
 ```python literalinclude emphasize-lines=4-10 caption=repository.py
-file:/dagster_examples/stocks/simple_partitions.py
+file:/legacy_examples/dagster_examples/stocks/simple_partitions.py
 lines:1-10
 ```
 
@@ -32,14 +32,14 @@ to the `query_historical_stock_data` solid config, just like we did in
 the previous section.
 
 ```python literalinclude caption=repository.py
-file:/dagster_examples/stocks/simple_partitions.py
+file:/legacy_examples/dagster_examples/stocks/simple_partitions.py
 lines:13-16
 ```
 
 Finally, we add the partitions to our repository definition:
 
 ```python literalinclude emphasize-lines=6
-file:/dagster_examples/stocks/repository.py
+file:/legacy_examples/dagster_examples/stocks/repository.py
 caption:repository.py
 lines:43-49
 ```

docs/next/src/pages/docs/learn/guides/partitioning/schedule_partitions.mdx

@@ -41,13 +41,13 @@ We can use the monthly decorator to create a partition set and schedule
 definition for our pipeline:
 
 ```python literalinclude caption=schedules.py
-file:/dagster_examples/stocks/schedules.py
+file:/legacy_examples/dagster_examples/stocks/schedules.py
 ```
 
 Then add the schedule definition to our repository
 
 ```python literalinclude emphasize-lines=5
-file:/dagster_examples/stocks/repository.py
+file:/legacy_examples/dagster_examples/stocks/repository.py
 caption:repository.py
 lines:43-49
 ```