To configure your first Generic Extractor, follow our tutorial.
To get an overall idea of what to expect when configuring Generic Extractor, take a look at the following overview of various configuration sections.
Then go through a sample configuration featuring all configuration options and their nesting. The configuration map is also available as a separate article.
Click on the section names if you want to learn more.
crt
/pem
format.crt
/pem
format.There are also simple pre-defined functions available, adding extra flexibility when needed.
Generic Extractor can be run from within the KBC user interface (only configuration JSON needed), or locally (Docker needed).
The following sample configuration shows various configuration options and their nesting. You can use the map to navigate between them. The parameter map is also available separately and we recommend pinning it to your toolbar for quick reference.
{
"parameters": {
"api": {
"baseUrl": "https://example.com/v3.0/",
"caCertificate": "-----BEGIN CERTIFICATE-----\nMIIFaz....",
"pagination": {
"method": "multiple",
"scrollers": {
"offset_scroll": {
"method": "offset",
"offsetParam": "offset",
"limitParam": "count"
}
}
},
"authentication": {
"type": "basic"
},
"retryConfig": {
"maxRetries": 3
},
"http": {
"headers": {
"Accept": "application/json"
},
"defaultOptions": {
"params": {
"company": 123
}
},
"requiredHeaders": ["X-AppKey"],
"ignoreErrors": [405],
"connectTimeout": 30,
"requestTimeout": 300
}
},
"aws": {
"signature": {
"credentials": {
"accessKeyId": "testAccessKey",
"#secretKey": "testSecretKey",
"serviceName": "testService",
"regionName": "testRegion"
}
}
},
"config": {
"debug": true,
"username": "dummy",
"#password": "secret",
"outputBucket": "ge-tutorial",
"incrementalOutput": true,
"compatLevel": 2,
"http": {
"headers": {
"X-AppKey": "ThisIsSecret"
}
},
"jobs": [
{
"endpoint": "users",
"method": "get",
"dataField": "items",
"dataType": "users",
"params": {
"type": {
"attr": "userType"
}
},
"responseFilter": "additional.address/details",
"responseFilterDelimiter": "/",
"scroller": "offset_scroll",
"children": [
{
"endpoint": "users/{user_id}/orders",
"dataField": "items",
"recursionFilter": "id>20",
"placeholders": {
"user_id": "id"
}
}
]
}
],
"mappings": {
"content": {
"parent_id": {
"type": "user",
"mapping": {
"destination": "campaign_id",
"primaryKey": true
}
},
"name": {
"type": "column",
"mapping": {
"destination": "text"
}
},
"address": {
"type": "table",
"destination": "addresses",
"tableMapping": {
"street": {
"type": "column",
"mapping": {
"destination": "streetName"
}
}
}
}
}
},
"userData": {
"tag": "development"
}
},
"iterations": [
{
"userType": "active"
},
{
"userType": "inactive"
}
],
"sshProxy": {
"host": "proxy.example.com",
"user": "proxy",
"port": 22,
"#privateKey": "-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----"
}
},
"authorization": {
"oauth_api": {
"credentials": {
"#data": "{\"status\": \"ok\",\"refresh_token\": \"1234abcd5678efgh\"}",
"appKey": "someId",
"#appSecret": "clientSecret"
}
}
}
}