To configure your first Generic Extractor, follow our tutorial.
Use the following sample configuration to navigate among various configuration options:
{ "parameters": { "api": { "baseUrl": "https://example.com/v3.0/", "caCertificate": "-----BEGIN CERTIFICATE-----\nMIIFaz....", "pagination": { "method": "multiple", "scrollers": { "offset_scroll": { "method": "offset", "offsetParam": "offset", "limitParam": "count" } } }, "authentication": { "type": "basic" }, "retryConfig": { "maxRetries": 3 }, "http": { "headers": { "Accept": "application/json" }, "defaultOptions": { "params": { "company": 123 } }, "requiredHeaders": ["X-AppKey"], "ignoreErrors": [405], "connectTimeout": 30, "requestTimeout": 300 } }, "aws": { "signature": { "credentials": { "accessKeyId": "testAccessKey", "#secretKey": "testSecretKey", "serviceName": "testService", "regionName": "testRegion" } } }, "config": { "debug": true, "username": "dummy", "#password": "secret", "outputBucket": "ge-tutorial", "incrementalOutput": true, "compatLevel": 2, "http": { "headers": { "X-AppKey": "ThisIsSecret" } }, "jobs": [ { "endpoint": "users", "method": "get", "dataField": "items", "dataType": "users", "params": { "type": { "attr": "userType" } }, "responseFilter": "additional.address/details", "responseFilterDelimiter": "/", "scroller": "offset_scroll", "children": [ { "endpoint": "users/{user_id}/orders", "dataField": "items", "recursionFilter": "id>20", "placeholders": { "user_id": "id" } } ] } ], "mappings": { "content": { "parent_id": { "type": "user", "mapping": { "destination": "campaign_id", "primaryKey": true } }, "name": { "type": "column", "mapping": { "destination": "text" } }, "address": { "type": "table", "destination": "addresses", "tableMapping": { "street": { "type": "column", "mapping": { "destination": "streetName" } } } }, "created.date": { "delimiter": "/", "type": "column", "mapping": { "destination": "createdDate" } } } }, "userData": { "tag": "development" } }, "iterations": [ { "userType": "active" }, { "userType": "inactive" } ], "sshProxy": { "host": "proxy.example.com", "user": "proxy", "port": 22, "#privateKey": "-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" } }, "authorization": { "oauth_api": { "credentials": { "#data": "{\"status\": \"ok\",\"refresh_token\": \"1234abcd5678efgh\"}", "appKey": "someId", "#appSecret": "clientSecret" } } } }