Getting started

meta

$ tar xzf metafacture-core-$VERSION-dist.tar.gz
$ cd metafacture-core-$VERSION
$ ./flux.sh examples/read/marc21/read-marc21.flux
<dependency>
    <groupId>org.metafacture</groupId>
    <artifactId>metamorph</artifactId>
    <version>$VERSION</version>
</dependency>
dependencies {
    implementation 'org.metafacture:metamorph:$VERSION'
}
$ git clone https://github.com/metafacture/metafacture-core.git
$ cd metafacture-core
$ ./gradlew publishToMavenLocal
$ ./gradlew assembleDist
# Fix is a macro-language for data transformations

# Simple fixes

add_field("hello", "world")
remove_field("my.deep.nested.junk")
copy_field("stats", "output.$append")

# Conditionals

if exists("error")
  set_field("is_valid", "no")
  log("error")
elsif exists("warning")
  set_field("is_valid", "yes")
  log("warning")
else
  set_field("is_valid", "yes")
end

# Loops

do list(path: "foo", "var": "$i")
  add_field("$i.bar", "baz")
end

include("<path>"[, <dynamicLocalVariables>...])
log("<logMessage>"[, level: "<logLevel>"])
nothing()
put_filemap("<sourceFile>", "<mapName>", sep_char: "\t")
put_map("<mapName>",
  "dog": "mammal",
  "parrot": "bird",
  "shark": "fish"
)
put_rdfmap("<rdfResource>", "<rdfMapName>", target: "<rdfProperty>")
put_rdfmap("<rdfResource>", "<rdfMapName>", target: "<rdfProperty>", select_language: "<rdfLanguageTag>")
put_var("<variableName>", "<variableValue>")
put_vars(
  "<variableName_1>": "<variableValue_1>",
  "<variableName_2>": "<variableValue_2>"
)
to_var("<sourceField>", "<variableName>")
add_array("<targetFieldName>")
add_array("<targetFieldName>", "<value_1>"[, ...])
add_field("<targetFieldName>", "<fieldValue>")
add_hash("<targetFieldName>")
add_hash("<targetFieldName>", "subfieldName": "<subfieldValue>"[, ...])
array("<sourceField>")
array("foo")
# {"name":"value"} => ["name", "value"]
do put_macro("<macroName>"[, <staticLocalVariables>...])
  ...
end
call_macro("<macroName>"[, <dynamicLocalVariables>...])
copy_field("<sourceField>", "<targetField>")
format("<sourceField>", "<formatString>")
hash("<sourceField>")
hash("foo")
# ["name", "value"] => {"name":"value"}
move_field("<sourceField>", "<targetField>")
parse_text("<sourceField>", "<parsePattern>")
paste("<targetField>", "<sourceField_1>"[, ...][, "join_char": ", "])
# a: eeny
# b: meeny
# c: miny
# d: moe
paste("my.string", "~Hi", "a", "~how are you?")
# "my.string": "Hi eeny how are you?"
print_record(["<prefix>"][, <options>...])
print_record("%d) Before transformation: ")
print_record(destination: "record-%2$s.json", id: "001", pretty: "true")
print_record(destination: "record-%03d.json.gz", header: "After transformation: ")
random("<targetField>", "<maximum>")
remove_field("<sourceField>")
rename("<sourceField>", "<regexp>", "<replacement>")
retain("<sourceField_1>"[, ...])
set_array("<targetFieldName>")
set_array("<targetFieldName>", "<value_1>"[, ...])
set_field("<targetFieldName>", "<fieldValue>")
set_hash("<targetFieldName>")
set_hash("<targetFieldName>", "subfieldName": "<subfieldValue>"[, ...])
timestamp("<targetField>"[, format: "<formatPattern>"][, timezone: "<timezoneCode>"][, language: "<languageCode>"])
vacuum()
append("<sourceField>", "<appendString>")
capitalize("<sourceField>")
count("<sourceField>")
downcase("<sourceField>")
filter("<sourceField>", "<regexp>")
flatten("<sourceField>")
from_json("<sourceField>"[, error_string: "<errorValue>"])
index("<sourceField>", "<substring>")
isbn("<sourceField>"[, to: "<isbnFormat>"][, verify_check_digit: "<boolean>"][, error_string: "<errorValue>"])
join_field("<sourceField>", "<separator>")
lookup("<sourceField>"[, <mapName>][, <options>...])
# local (unnamed) map
lookup("path.to.field", key_1: "value_1", ...)

# internal (named) map
put_map("internal-map", key_1: "value_1", ...)
lookup("path.to.field", "internal-map")

# external file map (implicit)
lookup("path.to.field", "path/to/file", sep_char: ";")

# external file map (explicit)
put_filemap("path/to/file", "file-map", sep_char: ";")
lookup("path.to.field", "file-map")

# RDF map (explicit)
put_rdfmap("path/to/file", "rdf-map", target: "<rdfProperty>")
lookup("path.to.field", "rdf-map")

# with default value
lookup("path.to.field", "map-name", "default": "NA")

# with printing unknown values to a file
lookup("path.to.field", "map-name", print_unknown: "true", destination: "unknown.txt")
prepend("<sourceField>", "<prependString>")
replace_all("<sourceField>", "<regexp>", "<replacement>")
reverse("<sourceField>")
sort_field("<sourceField>")
sort_field("<sourceField>", reverse: "true")
sort_field("<sourceField>", numeric: "true")
split_field("<sourceField>", "<separator>")
substring("<sourceField>", "<startPosition>", "<length>")
sum("<sourceField>")
to_json("<sourceField>"[, pretty: "<boolean>"][, error_string: "<errorValue>"])
to_base64("<sourceField>"[, url_safe: "<boolean>"])
trim("<sourceField>")
uniq("<sourceField>")
upcase("<sourceField>")
uri_encode("<sourceField>"[, <options>...])
uri_encode("path.to.field", plus_for_space:"false", safe_chars:"")
if <condition>
  reject()
end
do list(path: "<sourceField>")
  ...
end
do list(path: "<sourceField>", "var": "<variableName>")
  ...
end
do list_as(element_1: "<sourceField_1>"[, ...])
  ...
end
# "ccm:university":["https://ror.org/0304hq317"]
# "ccm:university_DISPLAYNAME":["Gottfried Wilhelm Leibniz Universität Hannover"]
set_array("sourceOrga[]")
do list_as(orgId: "ccm:university[]", orgName: "ccm:university_DISPLAYNAME[]")
  copy_field(orgId, "sourceOrga[].$append.id")
  copy_field(orgName, "sourceOrga[].$last.name")
end
# {"sourceOrga":[{"id":"https://ror.org/0304hq317","name":"Gottfried Wilhelm Leibniz Universität Hannover"}]}
do once()
  ...
end
do once("maps setup")
  ...
end
do once("vars setup")
  ...
end
do put_macro("<macroName>"[, <staticLocalVariables>...])
  ...
end
call_macro("<macroName>"[, <dynamicLocalVariables>...])
if <condition(params, ...)>
  ...
end
unless <condition(params, ...)>
  ...
end
if <condition(params, ...)>
  ...
elsif
  ...
else
  ...
end
if exists("<sourceField>")

Wildcard	Meaning
`*`	Selects all elements of an array.
`$first`	Selects only the first element of an array.
`$last`	Selects only the last element of an array.
`$prepend`	Selects the position before the first element of an array. Can only be used when adding new elements to an array.
`$append`	Selects the position after the last element of an array. Can only be used when adding new elements to an array.

Wildcard	Meaning
`*`	Placeholder for zero or more characters.
`?`	Placeholder for exactly one character.
`\|`	Alternation of multiple patterns.
`[...]`	Enumeration of characters.

Type	Separator
CSV	`,` or `;`
TSV	`\t`

Getting started

Metafacture as a stand-alone application

Using Metafacture as a Java library

Building metafacture-core from source

Metafacture Fix

Usage

Extension

Web editor

Functions and cookbook

Best practices and guidelines for working with Metafacture Fix

Glossary

Array wildcards

Path wildcards

Functions

Script-level functions

include

log

nothing

put_filemap

put_map

put_rdfmap

put_var

put_vars

to_var

Record-level functions

add_array

add_field

add_hash

array

call_macro

copy_field

format

hash

move_field

parse_text

paste

print_record

random

remove_field

rename

retain

set_array

set_field

set_hash

timestamp

vacuum

Field-level functions

append

capitalize

count

downcase

filter

flatten

from_json

index

isbn

join_field

lookup

prepend

replace_all

reverse

sort_field

split_field

substring

sum

to_json

to_base64

trim

uniq

upcase

uri_encode

Selectors

reject

Binds

do list

do list_as

do once

do put_macro

Conditionals

contain

`include`

`log`

`nothing`

`put_filemap`

`put_map`

`put_rdfmap`

`put_var`

`put_vars`

`to_var`

`add_array`

`add_field`

`add_hash`

`array`

`call_macro`

`copy_field`

`format`

`hash`

`move_field`

`parse_text`

`paste`

`print_record`

`random`

`remove_field`

`rename`

`retain`

`set_array`

`set_field`

`set_hash`

`timestamp`

`vacuum`

`append`

`capitalize`

`count`

`downcase`

`filter`

`flatten`

`from_json`

`index`

`isbn`

`join_field`

`lookup`

`prepend`

`replace_all`

`reverse`

`sort_field`

`split_field`

`substring`

`sum`

`to_json`

`to_base64`

`trim`

`uniq`

`upcase`

`uri_encode`

`reject`

`do list`

`do list_as`

`do once`

`do put_macro`

`contain`

`all_contain`

`any_contain`

`none_contain`

`str_contain`

`equal`

`all_equal`

`any_equal`

`none_equal`

`str_equal`

`exists`

`greater_than`

`in`

`is_contained_in`

`is_array`

`is_empty`

`is_false`

`is_hash`

`is_number`

`is_object`

`is_string`

License

metafacture/metafacture-core

About

Topics

Resources

License

Contributing

Uh oh!

Stars

Watchers

Forks

Releases 45

Packages 0

Uh oh!

Uh oh!

Contributors 21

Uh oh!

Languages

Packages