diff --git a/extensions/omniv21/fileformat/flatfile/fixedlength/decl.go b/extensions/omniv21/fileformat/flatfile/fixedlength/decl.go index 7b1afdb..c711f8f 100644 --- a/extensions/omniv21/fileformat/flatfile/fixedlength/decl.go +++ b/extensions/omniv21/fileformat/flatfile/fixedlength/decl.go @@ -65,16 +65,18 @@ const ( // If Header is specified, Rows must be nil. (JSON schema validation will ensure this.) // Footer is optional; If not specified, Header will be used for a single-line envelope matching. type EnvelopeDecl struct { - Name string `json:"name,omitempty"` - Rows *int `json:"rows,omitempty"` - Header *string `json:"header,omitempty"` - Footer *string `json:"footer,omitempty"` - Type *string `json:"type,omitempty"` - IsTarget bool `json:"is_target,omitempty"` - Min *int `json:"min,omitempty"` - Max *int `json:"max,omitempty"` - Columns []*ColumnDecl `json:"columns,omitempty"` - Children []*EnvelopeDecl `json:"child_envelopes,omitempty"` + Name string `json:"name,omitempty"` + Rows *int `json:"rows,omitempty"` + Header *string `json:"header,omitempty"` + HeaderNeg bool `json:"header_negative_match,omitempty"` + Footer *string `json:"footer,omitempty"` + FooterNeg bool `json:"footer_negative_match,omitempty"` + Type *string `json:"type,omitempty"` + IsTarget bool `json:"is_target,omitempty"` + Min *int `json:"min,omitempty"` + Max *int `json:"max,omitempty"` + Columns []*ColumnDecl `json:"columns,omitempty"` + Children []*EnvelopeDecl `json:"child_envelopes,omitempty"` fqdn string // fully hierarchical name to the envelope. childRecDecls []flatfile.RecDecl @@ -143,7 +145,7 @@ func (e *EnvelopeDecl) matchHeader(line []byte) bool { if e.headerRegexp == nil { panic(fmt.Sprintf("envelope '%s' is not header/footer based", e.fqdn)) } - return e.headerRegexp.Match(line) + return e.headerRegexp.Match(line) != e.HeaderNeg } // Footer is optional. If not specified, it always matches. Thus for a header/footer envelope, @@ -153,7 +155,7 @@ func (e *EnvelopeDecl) matchFooter(line []byte) bool { if e.footerRegexp == nil { return true } - return e.footerRegexp.Match(line) + return e.footerRegexp.Match(line) != e.FooterNeg } func toFlatFileRecDecls(es []*EnvelopeDecl) []flatfile.RecDecl { diff --git a/extensions/omniv21/samples/fixedlength2/3_header_footer.input.txt b/extensions/omniv21/samples/fixedlength2/3_header_footer.input.txt index 0973e4d..48b61de 100644 --- a/extensions/omniv21/samples/fixedlength2/3_header_footer.input.txt +++ b/extensions/omniv21/samples/fixedlength2/3_header_footer.input.txt @@ -1,89 +1,11 @@ A010 20191105 A011 044631 A020 210 -A030 201911051128 -A050 10583946 -A060 PostNL Pakketservice -A999 V010 V -V020 3SDMNN0129791 -V035 1400126111 -V040 3SDMNN0129791 -V046 201911051545 -V047 201911051815 -V060 J -V070 10 -V080 20191105 -V081 043523 -V090 Depot Nieuwegein -V091 Pakkettendistributie -V100 Depot Nieuwegein -V110 19780 -V120 132396 -V140 JAMES BOND -V150 123 -V160 3436JA -V170 WATERGATE -V180 NIEUWEGEIN -V200 NE -V999 -V010 V -V020 3SDMNN0129552 -V035 1399958395 -V040 3SDMNN0129552 -V045 20160825 -V046 201911051130 -V047 201911051400 -V060 M -V070 02 -V080 20191105 -V081 033000 V090 888888 -V091 Pakkettendistributie -V100 888888 -V110 19780 -V140 CAPTAIN AMERICA -V150 45 -V160 5211EK -V170 STAR VALLEY -V180 'S-HERTOGENBOSCH -V200 NL -V999 -V010 V -V020 3SDMNN0129857 -V035 1400185435 -V040 3SDMNN0129857 -V045 20160825 -V046 201911050800 -V047 201911051030 -V060 M -V070 02 -V080 20191105 -V081 033000 -V090 888888 -V091 Pakkettendistributie -V100 888888 -V110 1234 -V140 Stuart Little -V150 567 -V160 6871ZS -V170 MICKEYMOUSE -V180 RENKUM -V200 NE -V999 -V010 V -V020 3SDMNN0129501 -V035 1399918683 V040 3SDMNN0129501 V046 201911051515 V047 201911051745 -V060 M -V070 02 -V080 20191105 -V081 033000 -V090 888888 -V091 Pakkettendistributie -V100 888888 V140 JOHN WICK V150 9 V160 3043ME @@ -91,5 +13,4 @@ V170 CONTINENTAL HOTEL V180 ROTTERDAM V200 NL V999 -Z001 85 -Z999 +Z001 88 diff --git a/extensions/omniv21/samples/fixedlength2/3_header_footer.schema.json b/extensions/omniv21/samples/fixedlength2/3_header_footer.schema.json index 3b9ffd4..594b0e0 100644 --- a/extensions/omniv21/samples/fixedlength2/3_header_footer.schema.json +++ b/extensions/omniv21/samples/fixedlength2/3_header_footer.schema.json @@ -5,88 +5,18 @@ }, "file_declaration": { "envelopes" : [ - { - "name": "HEADER", "min": 1, "max": 1, "header": "^A010", "footer": "^A999", - "columns": [ { "name": "carrier", "start_pos": 6, "length": 6, "line_pattern": "^A060" } ] - }, - { - "name": "REC", "header": "^V010", "footer": "^V999", "is_target": true, + { "name": "HEADER", "min": 1, "max": 1, "rows": 3 }, + { "name": "MIDDLE", "header": "^Z001", "header_negative_match": true }, + { "name": "FOOTER", "min": 1, "max": 1, "header": "^Z001", "is_target": true, "columns": [ - { "name": "tracking_number", "start_pos": 6, "length": 15, "line_pattern": "^V020" }, - { "name": "delivery_date", "start_pos": 6, "length": 8, "line_pattern": "^V045" }, - { "name": "observation_type", "start_pos": 6, "length": 1, "line_pattern": "^V060" }, - { "name": "reason_for_observation", "start_pos": 6, "length": 2, "line_pattern": "^V070" }, - { "name": "date_observation", "start_pos": 6, "length": 8, "line_pattern": "^V080" }, - { "name": "time_observation", "start_pos": 6, "length": 6, "line_pattern": "^V081" }, - { "name": "weight_in_grams", "start_pos": 6, "length": 6, "line_pattern": "^V110" }, - { "name": "postal_code_addressee", "start_pos": 6, "length": 6, "line_pattern": "^V160" }, - { "name": "city_name_addressee", "start_pos": 6, "length": 24, "line_pattern": "^V180" }, - { "name": "country_code_addressee", "start_pos": 6, "length": 2, "line_pattern": "^V200" } + { "name": "age", "start_pos": 6, "length": 2, "line_pattern": "^Z001" } ] - }, - { "name": "FOOTER", "min": 1, "max": 1, "header": "^Z001", "footer": "^Z999" } + } ] }, "transform_declarations": { "FINAL_OUTPUT": { "object": { - "tracking_number": { "xpath": "tracking_number" }, - "carrier": { "custom_func": { "name": "lower", "args": [ { "xpath": "../HEADER/carrier" } ] } }, - "estimated_delivery_date": { "custom_func": { - "name": "dateTimeToRFC3339", - "args": [ - { "xpath": "delivery_date" }, - { "const": "", "_comment": "input timezone" }, - { "const": "", "_comment": "output timezone" } - ] - }}, - "weight_in_kg": { - "custom_func": { - "name": "javascript", - "args": [ - { "const": "((/^[0-9]+$/.test(grams) ? parseFloat(grams) : 0.0) / 1000).toFixed(2)" }, - { "const": "grams" }, { "xpath": "weight_in_grams" } - ] - }, - "type": "float" - }, - "events": { "array": [ - { "object": { - "event_date": { "template": "event_date_template" }, - "location": { "object": { - "city": { "custom_func": { - "name": "javascript", - "args": [ - { "const": "event_country == 'NL' ? event_city : ''" }, - { "const": "event_country"}, { "xpath": "country_code_addressee" }, - { "const": "event_city"}, { "xpath": "city_name_addressee" } - ] - }}, - "zip": { "custom_func": { - "name": "javascript", - "args": [ - { "const": "event_country == 'NL' ? event_zip : '0000FF'" }, - { "const": "event_country"}, { "xpath": "country_code_addressee" }, - { "const": "event_zip"}, { "xpath": "postal_code_addressee" } - ] - }}, - "country": { "xpath": "country_code_addressee" } - }} - }} - ]} - }}, - "event_date_template": { "custom_func": { - "name": "dateTimeToRFC3339", - "args": [ - { "custom_func": { - "name": "concat", - "args": [ - { "xpath": "date_observation" }, - { "xpath": "time_observation" } - ] - }}, - { "const": "", "_comment": "input timezone" }, - { "const": "", "_comment": "output timezone" } - ] + "age": { "xpath": "age" } }} } } diff --git a/extensions/omniv21/validation/csv2FileDeclaration.go b/extensions/omniv21/validation/csv2FileDeclaration.go index 11b950a..6fcd9e9 100644 --- a/extensions/omniv21/validation/csv2FileDeclaration.go +++ b/extensions/omniv21/validation/csv2FileDeclaration.go @@ -68,7 +68,9 @@ const ( "properties": { "name": { "type": "string", "minLength": 1 }, "header": { "type": "string", "minLength": 1 }, + "header_negative_match": { "type": "boolean" }, "footer": { "type": "string", "minLength": 1 }, + "footer_negative_match": { "type": "boolean" }, "type": { "const": "record" }, "is_target": { "type": "boolean" }, "min": { "type": "integer", "minimum": 0 }, diff --git a/extensions/omniv21/validation/csv2FileDeclaration.json b/extensions/omniv21/validation/csv2FileDeclaration.json index 185715e..8755c66 100644 --- a/extensions/omniv21/validation/csv2FileDeclaration.json +++ b/extensions/omniv21/validation/csv2FileDeclaration.json @@ -61,7 +61,9 @@ "properties": { "name": { "type": "string", "minLength": 1 }, "header": { "type": "string", "minLength": 1 }, + "header_negative_match": { "type": "boolean" }, "footer": { "type": "string", "minLength": 1 }, + "footer_negative_match": { "type": "boolean" }, "type": { "const": "record" }, "is_target": { "type": "boolean" }, "min": { "type": "integer", "minimum": 0 }, diff --git a/extensions/omniv21/validation/fixedlength2FileDeclaration.go b/extensions/omniv21/validation/fixedlength2FileDeclaration.go index 6a71d5b..8fe7996 100644 --- a/extensions/omniv21/validation/fixedlength2FileDeclaration.go +++ b/extensions/omniv21/validation/fixedlength2FileDeclaration.go @@ -66,7 +66,9 @@ const ( "properties": { "name": { "type": "string", "minLength": 1 }, "header": { "type": "string", "minLength": 1 }, + "header_negative_match": { "type": "boolean" }, "footer": { "type": "string", "minLength": 1 }, + "footer_negative_match": { "type": "boolean" }, "type": { "const": "envelope" }, "is_target": { "type": "boolean" }, "min": { "type": "integer", "minimum": 0 }, diff --git a/extensions/omniv21/validation/fixedlength2FileDeclaration.json b/extensions/omniv21/validation/fixedlength2FileDeclaration.json index a199416..24453e1 100644 --- a/extensions/omniv21/validation/fixedlength2FileDeclaration.json +++ b/extensions/omniv21/validation/fixedlength2FileDeclaration.json @@ -59,7 +59,9 @@ "properties": { "name": { "type": "string", "minLength": 1 }, "header": { "type": "string", "minLength": 1 }, + "header_negative_match": { "type": "boolean" }, "footer": { "type": "string", "minLength": 1 }, + "footer_negative_match": { "type": "boolean" }, "type": { "const": "envelope" }, "is_target": { "type": "boolean" }, "min": { "type": "integer", "minimum": 0 },