Remove old references (apache#290)

chenjian2664 · Sep 13, 2024 · 44fdc57 · 44fdc57
1 parent 874d866
commit 44fdc57
Show file tree

Hide file tree

Showing 7 changed files with 106 additions and 86 deletions.
diff --git a/docs/access-control.md b/docs/access-control.md
@@ -160,8 +160,8 @@ includes the following users:
     create service principals. She can also create catalogs and
     namespaces and configure access control for Polaris resources.
 
-- **Bob:** A data engineer who uses Snowpipe Streaming (in Snowflake)
-    and Apache Spark&trade; connections to interact with Polaris.
+- **Bob:** A data engineer who uses Apache Spark&trade; to 
+    interact with Polaris.
 
     - Alice has created a service principal for Bob. It has been
         granted the Data_engineer principal role, which in turn has been
@@ -175,8 +175,8 @@ includes the following users:
     - The Data administrator roles grant full administrative rights to
         the Silver zone catalog and Gold zone catalog.
 
-- **Mark:** A data scientist who uses Snowflake AI services to
-    interact with Polaris.
+- **Mark:** A data scientist who uses trains models with data managed
+    by Polaris.
 
     - Alice has created a service principal for Mark. It has been
         granted the Data_scientist principal role, which in turn has

diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/overview.md b/docs/overview.md
@@ -68,10 +68,10 @@ nested namespaces. Iceberg tables belong to namespaces.
 In an internal catalog, an Iceberg table is registered in Polaris, but read and written via query engines. The table data and
 metadata is stored in your external cloud storage. The table uses Polaris as the Iceberg catalog.
 
-If you have tables that use Snowflake as the Iceberg catalog (Snowflake-managed tables), you can sync these tables to an external
-catalog in Polaris. If you sync this catalog to Polaris, it appears as an external catalog in Polaris. The table data and
-metadata is stored in your external cloud storage. The Snowflake query engine can read from or write to these tables. However, the other query
-engines can only read from these tables.
+If you have tables housed in another Iceberg catalog, you can sync these tables to an external catalog in Polaris.
+If you sync this catalog to Polaris, it appears as an external catalog in Polaris. Clients connecting to the external
+catalog can read from or write to these tables. However, clients connecting to Polaris will only be able to
+read from these tables.
 
 > **Important**
 >
@@ -156,12 +156,11 @@ In the following example workflow, Bob creates an Apache Iceberg&trade; table na
     service connection with a service principal that has
     the privileges to perform these actions.
 
-2.  Alice uses Snowflake to read data from Table1.
+2.  Alice uses Trino to read data from Table1.
 
     Alice can read data from Table1 because she is using a service
-    connection with a service principal with a catalog integration that
-    has the privileges to perform this action. Alice
-    creates an unmanaged table in Snowflake to read data from Table1.
+    connection with a service principal that has the privileges to 
+    perform this action.
 
 ![Diagram that shows an example workflow for Apache Polaris (Incubating)](img/example-workflow.svg "Example workflow for Apache Polaris (Incubating)")
 

diff --git a/polaris-server.yml b/polaris-server.yml
@@ -73,10 +73,6 @@ featureConfiguration:
     - AZURE
     - FILE
 
-
-# Whether we want to enable Snowflake OAuth locally. Setting this to true requires
-# that you go through the setup outlined in the `README.md` file, specifically the
-# `OAuth + Snowflake: Local Testing And Then Some` section
 callContextResolver:
   type: default
 
@@ -162,8 +158,8 @@ logging:
 
       # The file to which statements will be logged.
       currentLogFilename: ./logs/polaris.log
-      #  When the log file rolls over, the file will be archived to snowflake-2012-03-15.log.gz,
-      # snowflake.log will be truncated, and new statements written to it.
+      #  When the log file rolls over, the file will be archived to polaris-2012-03-15.log.gz,
+      # polaris.log will be truncated, and new statements written to it.
       archivedLogFilenamePattern: ./logs/polaris-%d.log.gz
       # The maximum number of log files to archive.
       archivedFileCount: 14
diff --git a/polaris-service/src/test/resources/polaris-server-integrationtest.yml b/polaris-service/src/test/resources/polaris-server-integrationtest.yml
@@ -80,10 +80,6 @@ featureConfiguration:
 
 metaStoreManager:
   type: in-memory
-#  type: remote
-#  url: http://sdp-devvm-mcollado:8080
-# type: eclipse-link # uncomment to use eclipse-link as metastore
-# persistence-unit: polaris
 
 io:
   factoryType: default
@@ -93,10 +89,6 @@ oauth2:
   tokenBroker:
     type: symmetric-key
     secret: polaris
-#  type: snowflake
-#  clientId: ${GS_POLARIS_SERVICE_CLIENT_ID}
-#  clientSecret: ${GS_POLARIS_SERVICE_CLIENT_SECRET}
-#  clientSecret2: ${GS_POLARIS_SERVICE_CLIENT_SECRET2}
 
 authenticator:
   class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator
@@ -107,25 +99,15 @@ authenticator:
 
 callContextResolver:
   type: default
-#  type: snowflake
-#  account: ${SNOWFLAKE_ACCOUNT:-SNOWFLAKE}
-#  scheme: ${GS_SCHEME:-http}
-#  host: ${GS_HOST:-localhost}
-#  port: ${GS_PORT:-8080}
 
 realmContextResolver:
   type: default
-#  type: snowflake
-#  account: ${SNOWFLAKE_ACCOUNT:-SNOWFLAKE}
-#  scheme: ${GS_SCHEME:-http}
-#  host: ${GS_HOST:-localhost}
-#  port: ${GS_PORT:-8080}
 
-defaultRealm: SNOWFLAKE
+defaultRealm: POLARIS
 
 cors:
   allowed-origins:
-    - snowflake.com
+    - localhost
 
   # Logging settings.
 logging:
@@ -162,8 +144,8 @@ logging:
 
       # The file to which statements will be logged.
       currentLogFilename: ./logs/iceberg-rest.log
-      #  When the log file rolls over, the file will be archived to snowflake-2012-03-15.log.gz,
-      # snowflake.log will be truncated, and new statements written to it.
+      # When the log file rolls over, the file will be archived to polaris-2012-03-15.log.gz,
+      # polaris.log will be truncated, and new statements written to it.
       archivedLogFilenamePattern: ./logs/iceberg-rest-%d.log.gz
       # The maximum number of log files to archive.
       archivedFileCount: 14
diff --git a/regtests/t_pyspark/src/iceberg_spark.py b/regtests/t_pyspark/src/iceberg_spark.py
@@ -35,7 +35,7 @@ class IcebergSparkSession:
           polaris_url="http://polaris:8181/api/catalog",
           catalog_name="catalog_name"
   ) as spark:
-      spark.sql(f"USE snowflake.{hybrid_executor.database}.{hybrid_executor.schema}")
+      spark.sql(f"USE catalog.{hybrid_executor.database}.{hybrid_executor.schema}")
       table_list = spark.sql("SHOW TABLES").collect()
   """
 

diff --git a/spec/index.yaml b/spec/index.yaml
@@ -168,16 +168,16 @@ tags:
       as the Iceberg catalog.
 
 
-      If you have tables that use Snowflake as the Iceberg catalog
-      (Snowflake-managed tables), you can sync these tables to an external
+      If you have tables housed in another Iceberg catalog, you can sync these
+      tables to an external catalog in Polaris.
 
-      catalog in Polaris. If you sync this catalog to Polaris, it appears as an
-      external catalog in Polaris. The table data and
+      If you sync this catalog to Polaris, it appears as an external catalog in
+      Polaris. Clients connecting to the external
 
-      metadata is stored in your external cloud storage. The Snowflake query
-      engine can read from or write to these tables. However, the other query
+      catalog can read from or write to these tables. However, clients
+      connecting to Polaris will only be able to
 
-      engines can only read from these tables.
+      read from these tables.
 
 
       > **Important**
@@ -348,12 +348,11 @@ tags:
           service connection with a service principal that has
           the privileges to perform these actions.
 
-      2.  Alice uses Snowflake to read data from Table1.
+      2.  Alice uses Trino to read data from Table1.
 
           Alice can read data from Table1 because she is using a service
-          connection with a service principal with a catalog integration that
-          has the privileges to perform this action. Alice
-          creates an unmanaged table in Snowflake to read data from Table1.
+          connection with a service principal that has the privileges to 
+          perform this action.
 
       ![Diagram that shows an example workflow for Apache Polaris
       (Incubating)](img/example-workflow.svg "Example workflow for Apache
@@ -918,8 +917,8 @@ tags:
           create service principals. She can also create catalogs and
           namespaces and configure access control for Polaris resources.
 
-      - **Bob:** A data engineer who uses Snowpipe Streaming (in Snowflake)
-          and Apache Spark&trade; connections to interact with Polaris.
+      - **Bob:** A data engineer who uses Apache Spark&trade; to 
+          interact with Polaris.
 
           - Alice has created a service principal for Bob. It has been
               granted the Data_engineer principal role, which in turn has been
@@ -933,8 +932,8 @@ tags:
           - The Data administrator roles grant full administrative rights to
               the Silver zone catalog and Gold zone catalog.
 
-      - **Mark:** A data scientist who uses Snowflake AI services to
-          interact with Polaris.
+      - **Mark:** A data scientist who uses trains models with data managed
+          by Polaris.
 
           - Alice has created a service principal for Mark. It has been
               granted the Data_scientist principal role, which in turn has
@@ -3106,7 +3105,16 @@ paths:
       summary: Sends a notification to the table
       operationId: sendNotification
       requestBody:
-        description: The request containing the notification to be sent
+        description: >-
+          The request containing the notification to be sent.
+
+          For each table, Polaris will reject any notification where the
+          timestamp in the request body is older than or equal to the most
+          recent time Polaris has already processed for the table. The
+          responsibility of ensuring the correct order of timestamps for a
+          sequence of notifications lies with the caller of the API. This
+          includes managing potential clock skew or inconsistencies when
+          notifications are sent from multiple sources.
         content:
           application/json:
             schema:
@@ -3136,6 +3144,28 @@ paths:
                 TableToLoadDoesNotExist:
                   $ref: >-
                     #/components/examples/Apache_Iceberg_REST_Catalog_API_NoSuchTableError
+        '409':
+          description: >-
+            Conflict - The timestamp of the received notification is older than
+            or equal to the most recent timestamp Polaris has already processed
+            for the specified table.
+          content:
+            application/json:
+              schema:
+                $ref: >-
+                  #/components/schemas/Apache_Iceberg_REST_Catalog_API_IcebergErrorResponse
+              example:
+                summary: >-
+                  The timestamp of the received notification is older than or
+                  equal to the most recent timestamp Polaris has already
+                  processed for the specified table.
+                value:
+                  error:
+                    message: >-
+                      A notification with a newer timestamp has been admitted
+                      for table
+                    type: AlreadyExistsException
+                    code: 409
         '419':
           $ref: >-
             #/components/responses/Apache_Iceberg_REST_Catalog_API_AuthenticationTimeoutResponse
@@ -6044,6 +6074,7 @@ components:
     Apache_Iceberg_REST_Catalog_API_NotificationRequest:
       required:
         - notification-type
+        - payload
       properties:
         notification-type:
           $ref: >-
@@ -7016,6 +7047,16 @@ components:
           - bar
         updates:
           owner: Raoul
+    Apache_Iceberg_REST_Catalog_API_OutOfOrderNotificationError:
+      summary: >-
+        The timestamp of the received notification is older than or equal to the
+        most recent timestamp Polaris has already processed for the specified
+        table.
+      value:
+        error:
+          message: A notification with a newer timestamp has been admitted for table
+          type: AlreadyExistsException
+          code: 409
 x-tagGroups:
   - name: Apache Polaris (Incubating) Documentation
     tags: