forked from logicalclocks/feature-store-api
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathauto_doc.py
421 lines (413 loc) · 17.3 KB
/
auto_doc.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
import pathlib
import shutil
import keras_autodoc
PAGES = {
"project.md": {
"connection": ["hsfs.connection.Connection"],
"connection_methods": keras_autodoc.get_methods(
"hsfs.connection.Connection", exclude=["connection"]
),
},
"feature_store.md": {
"fs_get": ["hsfs.connection.Connection.get_feature_store"],
"fs_properties": keras_autodoc.get_properties(
"hsfs.feature_store.FeatureStore"
),
"fs_methods": keras_autodoc.get_methods(
"hsfs.feature_store.FeatureStore", exclude=["from_response_json"]
),
},
"feature.md": {
"feature": ["hsfs.feature.Feature"],
"feature_properties": keras_autodoc.get_properties("hsfs.feature.Feature"),
"feature_methods": keras_autodoc.get_methods(
"hsfs.feature.Feature", exclude=["from_response_json", "to_dict"]
),
},
"feature_group.md": {
"fg_create": ["hsfs.feature_store.FeatureStore.create_feature_group"],
"fg_get": ["hsfs.feature_store.FeatureStore.get_feature_group"],
"fg_properties": keras_autodoc.get_properties(
"hsfs.feature_group.FeatureGroup"
),
"fg_methods": keras_autodoc.get_methods(
"hsfs.feature_group.FeatureGroup",
exclude=[
"from_response_json",
"update_from_response_json",
"json",
"to_dict",
],
),
},
"on_demand_feature_group.md": {
"fg_create": ["hsfs.feature_store.FeatureStore.create_on_demand_feature_group"],
"fg_get": ["hsfs.feature_store.FeatureStore.get_on_demand_feature_group"],
"fg_properties": keras_autodoc.get_properties(
"hsfs.feature_group.OnDemandFeatureGroup"
),
"fg_methods": keras_autodoc.get_methods(
"hsfs.feature_group.OnDemandFeatureGroup",
exclude=[
"from_response_json",
"update_from_response_json",
"json",
"to_dict",
],
),
},
"training_dataset.md": {
"td_create": ["hsfs.feature_store.FeatureStore.create_training_dataset"],
"td_get": ["hsfs.feature_store.FeatureStore.get_training_dataset"],
"td_properties": keras_autodoc.get_properties(
"hsfs.training_dataset.TrainingDataset"
),
"td_methods": keras_autodoc.get_methods(
"hsfs.training_dataset.TrainingDataset",
exclude=[
"from_response_json",
"update_from_response_json",
"json",
"to_dict",
],
),
},
"storage_connector.md": {
"sc_get": [
"hsfs.feature_store.FeatureStore.get_storage_connector",
"hsfs.feature_store.FeatureStore.get_online_storage_connector",
],
"hopsfs_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.HopsFSConnector", exclude=["from_response_json"]
),
"hopsfs_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.HopsFSConnector"
),
"s3_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.S3Connector", exclude=["from_response_json"]
),
"s3_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.S3Connector"
),
"redshift_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.RedshiftConnector", exclude=["from_response_json"]
),
"redshift_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.RedshiftConnector"
),
"adls_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.AdlsConnector", exclude=["from_response_json"]
),
"adls_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.AdlsConnector"
),
"snowflake_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.SnowflakeConnector", exclude=["from_response_json"]
),
"snowflake_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.SnowflakeConnector"
),
"jdbc_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.JdbcConnector", exclude=["from_response_json"]
),
"jdbc_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.JdbcConnector"
),
"gcs_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.GcsConnector", exclude=["from_response_json"]
),
"gcs_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.GcsConnector"
),
"bigquery_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.BigQueryConnector", exclude=["from_response_json"]
),
"bigquery_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.BigQueryConnector"
),
},
"query_vs_dataframe.md": {
"query_methods": keras_autodoc.get_methods("hsfs.constructor.query.Query"),
"query_properties": keras_autodoc.get_properties(
"hsfs.constructor.query.Query"
),
},
"statistics.md": {
"statistics_config": ["hsfs.statistics_config.StatisticsConfig"],
"statistics_config_properties": keras_autodoc.get_properties(
"hsfs.statistics_config.StatisticsConfig"
),
},
"feature_validation.md": {
"rule": ["hsfs.rule.Rule"],
"rule_properties": keras_autodoc.get_properties("hsfs.rule.Rule"),
"ruledefinition": ["hsfs.ruledefinition.RuleDefinition"],
"ruledefinition_getall": ["hsfs.connection.Connection.get_rules"],
"ruledefinition_get": ["hsfs.connection.Connection.get_rule"],
"ruledefinition_properties": keras_autodoc.get_properties(
"hsfs.ruledefinition.RuleDefinition"
),
"expectation": ["hsfs.expectation.Expectation"],
"expectation_properties": keras_autodoc.get_properties(
"hsfs.expectation.Expectation"
),
"expectation_methods": keras_autodoc.get_methods(
"hsfs.expectation.Expectation",
exclude=[
"from_response_json",
"update_from_response_json",
"json",
"to_dict",
],
),
"expectation_create": ["hsfs.feature_store.FeatureStore.create_expectation"],
"expectation_get": ["hsfs.feature_store.FeatureStore.get_expectation"],
"expectation_getall": ["hsfs.feature_store.FeatureStore.get_expectations"],
"validation_result": ["hsfs.validation_result.ValidationResult"],
"validation_result_properties": keras_autodoc.get_properties(
"hsfs.validation_result.ValidationResult"
),
"validate": ["hsfs.feature_group.FeatureGroup.validate"],
"validation_result_get": ["hsfs.feature_group.FeatureGroup.get_validations"],
},
"tags.md": {
"fg_tag_add": ["hsfs.feature_group.FeatureGroupBase.add_tag"],
"fg_tag_get": ["hsfs.feature_group.FeatureGroupBase.get_tag"],
"fg_tag_get_all": ["hsfs.feature_group.FeatureGroupBase.get_tags"],
"fg_tag_delete": ["hsfs.feature_group.FeatureGroupBase.delete_tag"],
"td_tag_add": ["hsfs.training_dataset.TrainingDataset.add_tag"],
"td_tag_get": ["hsfs.training_dataset.TrainingDataset.get_tag"],
"td_tag_get_all": ["hsfs.training_dataset.TrainingDataset.get_tags"],
"td_tag_delete": ["hsfs.training_dataset.TrainingDataset.delete_tag"],
},
"transformation_functions.md": {
"transformation_function": [
"hsfs.transformation_function.TransformationFunction"
],
"transformation_function_properties": keras_autodoc.get_properties(
"hsfs.transformation_function.TransformationFunction"
),
"transformation_function_methods": keras_autodoc.get_methods(
"hsfs.transformation_function.TransformationFunction",
exclude=[
"from_response_json",
"update_from_response_json",
"json",
"to_dict",
],
),
"create_transformation_function": [
"hsfs.feature_store.FeatureStore.create_transformation_function"
],
"get_transformation_function": [
"hsfs.feature_store.FeatureStore.get_transformation_function"
],
"get_transformation_functions": [
"hsfs.feature_store.FeatureStore.get_transformation_functions"
],
},
"api/connection_api.md": {
"connection": ["hsfs.connection.Connection"],
"connection_properties": keras_autodoc.get_properties(
"hsfs.connection.Connection"
),
"connection_methods": keras_autodoc.get_methods("hsfs.connection.Connection"),
},
"api/feature_store_api.md": {
"fs": ["hsfs.feature_store.FeatureStore"],
"fs_get": ["hsfs.connection.Connection.get_feature_store"],
"fs_properties": keras_autodoc.get_properties(
"hsfs.feature_store.FeatureStore"
),
"fs_methods": keras_autodoc.get_methods("hsfs.feature_store.FeatureStore"),
},
"api/feature_group_api.md": {
"fg": ["hsfs.feature_group.FeatureGroup"],
"fg_create": ["hsfs.feature_store.FeatureStore.create_feature_group"],
"fg_get": ["hsfs.feature_store.FeatureStore.get_feature_group"],
"fg_properties": keras_autodoc.get_properties(
"hsfs.feature_group.FeatureGroup"
),
"fg_methods": keras_autodoc.get_methods("hsfs.feature_group.FeatureGroup"),
},
"api/training_dataset_api.md": {
"td": ["hsfs.training_dataset.TrainingDataset"],
"td_create": ["hsfs.feature_store.FeatureStore.create_training_dataset"],
"td_get": ["hsfs.feature_store.FeatureStore.get_training_dataset"],
"td_properties": keras_autodoc.get_properties(
"hsfs.training_dataset.TrainingDataset"
),
"td_methods": keras_autodoc.get_methods(
"hsfs.training_dataset.TrainingDataset"
),
},
"api/feature_api.md": {
"feature": ["hsfs.feature.Feature"],
"feature_properties": keras_autodoc.get_properties("hsfs.feature.Feature"),
"feature_methods": keras_autodoc.get_methods("hsfs.feature.Feature"),
},
"api/storage_connector_api.md": {
"sc_get": [
"hsfs.feature_store.FeatureStore.get_storage_connector",
"hsfs.feature_store.FeatureStore.get_online_storage_connector",
],
"hopsfs_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.HopsFSConnector", exclude=["from_response_json"]
),
"hopsfs_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.HopsFSConnector"
),
"s3_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.S3Connector", exclude=["from_response_json"]
),
"s3_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.S3Connector"
),
"redshift_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.RedshiftConnector", exclude=["from_response_json"]
),
"redshift_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.RedshiftConnector"
),
"adls_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.AdlsConnector", exclude=["from_response_json"]
),
"adls_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.AdlsConnector"
),
"snowflake_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.SnowflakeConnector", exclude=["from_response_json"]
),
"snowflake_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.SnowflakeConnector"
),
"jdbc_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.JdbcConnector", exclude=["from_response_json"]
),
"jdbc_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.JdbcConnector"
),
"gcs_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.GcsConnector", exclude=["from_response_json"]
),
"gcs_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.GcsConnector"
),
"bigquery_methods": keras_autodoc.get_methods(
"hsfs.storage_connector.BigQueryConnector", exclude=["from_response_json"]
),
"bigquery_properties": keras_autodoc.get_properties(
"hsfs.storage_connector.BigQueryConnector"
),
},
"api/statistics_config_api.md": {
"statistics_config": ["hsfs.statistics_config.StatisticsConfig"],
"statistics_config_properties": keras_autodoc.get_properties(
"hsfs.statistics_config.StatisticsConfig"
),
},
"api/rule_api.md": {
"rule": ["hsfs.rule.Rule"],
"rule_properties": keras_autodoc.get_properties("hsfs.rule.Rule"),
},
"api/rule_definition_api.md": {
"ruledefinition": ["hsfs.ruledefinition.RuleDefinition"],
"ruledefinition_getall": ["hsfs.connection.Connection.get_rules"],
"ruledefinition_get": ["hsfs.connection.Connection.get_rule"],
"ruledefinition_properties": keras_autodoc.get_properties(
"hsfs.ruledefinition.RuleDefinition"
),
},
"api/expectation_api.md": {
"expectation": ["hsfs.expectation.Expectation"],
"expectation_properties": keras_autodoc.get_properties(
"hsfs.expectation.Expectation"
),
"expectation_methods": keras_autodoc.get_methods(
"hsfs.expectation.Expectation",
exclude=[
"from_response_json",
"update_from_response_json",
"json",
"to_dict",
],
),
"expectation_create": ["hsfs.feature_store.FeatureStore.create_expectation"],
"expectation_get": ["hsfs.feature_store.FeatureStore.get_expectation"],
"expectation_getall": ["hsfs.feature_store.FeatureStore.get_expectations"],
},
"api/validation_api.md": {
"validation_result": ["hsfs.validation_result.ValidationResult"],
"validation_result_properties": keras_autodoc.get_properties(
"hsfs.validation_result.ValidationResult"
),
"validate": ["hsfs.feature_group.FeatureGroup.validate"],
"validation_result_get": ["hsfs.feature_group.FeatureGroup.get_validations"],
},
"api/transformation_functions_api.md": {
"transformation_function": [
"hsfs.transformation_function.TransformationFunction"
],
"transformation_function_properties": keras_autodoc.get_properties(
"hsfs.transformation_function.TransformationFunction"
),
"transformation_function_methods": keras_autodoc.get_methods(
"hsfs.transformation_function.TransformationFunction",
exclude=[
"from_response_json",
"update_from_response_json",
"json",
"to_dict",
],
),
"create_transformation_function": [
"hsfs.feature_store.FeatureStore.create_transformation_function"
],
"get_transformation_function": [
"hsfs.feature_store.FeatureStore.get_transformation_function"
],
"get_transformation_functions": [
"hsfs.feature_store.FeatureStore.get_transformation_functions"
],
},
"api/job_configuration.md": {
"job_configuration": ["hsfs.core.job_configuration.JobConfiguration"]
},
"api/query_api.md": {
"query_methods": keras_autodoc.get_methods(
"hsfs.constructor.query.Query",
exclude=["json", "to_dict"],
),
"query_properties": keras_autodoc.get_properties(
"hsfs.constructor.query.Query"
),
},
"versioning.md": {
"fg_append": ["hsfs.feature_group.FeatureGroup.append_features"],
"fg_get_all": ["hsfs.feature_store.FeatureStore.get_feature_groups"],
"td_get_all": ["hsfs.feature_store.FeatureStore.get_training_datasets"],
"as_of": ["hsfs.constructor.query.Query.as_of"],
"commit_details": ["hsfs.feature_group.FeatureGroup.commit_details"],
},
}
hsfs_dir = pathlib.Path(__file__).resolve().parents[0]
def generate(dest_dir):
doc_generator = keras_autodoc.DocumentationGenerator(
PAGES,
project_url="https://github.com/logicalclocks/feature-store-api/blob/master/python",
template_dir="./docs/templates",
titles_size="###",
extra_aliases={
"hsfs.core.query.Query": "hsfs.Query",
"hsfs.storage_connector.StorageConnector": "hsfs.StorageConnector",
"hsfs.statistics_config.StatisticsConfig": "hsfs.StatisticsConfig",
"hsfs.training_dataset_feature.TrainingDatasetFeature": "hsfs.TrainingDatasetFeature",
"pandas.core.frame.DataFrame": "pandas.DataFrame",
},
max_signature_line_length=100,
)
shutil.copyfile(hsfs_dir / "CONTRIBUTING.md", dest_dir / "CONTRIBUTING.md")
shutil.copyfile(hsfs_dir / "README.md", dest_dir / "index.md")
doc_generator.generate(dest_dir / "generated")
if __name__ == "__main__":
generate(hsfs_dir / "docs")