node-es-transformer/examples/basic-file-ingestion.js at main · walterra/node-es-transformer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
/**
 * Basic File Ingestion Example
 *
 * Demonstrates how to ingest a JSON or CSV file into Elasticsearch
 * with custom field mappings.
 */

const transformer = require('node-es-transformer');
const logger = require('./_logger');

// Example 1: Ingest JSON file
transformer({
  fileName: 'data/sample.json',
  targetIndexName: 'my-data-index',
  mappings: {
    properties: {
      '@timestamp': {
        type: 'date',
      },
      user_name: {
        type: 'keyword',
      },
      message: {
        type: 'text',
      },
      count: {
        type: 'integer',
      },
    },
  },
})
  .then(() => {
    logger.info('Ingestion complete');
  })
  .catch(err => {
    logger.error({ err }, 'Error during ingestion');
  });

// Example 2: Ingest with custom Elasticsearch connection
/*
transformer({
  fileName: 'data/sample.json',
  targetIndexName: 'my-data-index',
  targetClientConfig: {
    node: 'https://elasticsearch.example.com:9200',
    auth: {
      apiKey: process.env.ES_API_KEY
    },
    tls: {
      rejectUnauthorized: true
    }
  },
  mappings: {
    properties: {
      '@timestamp': { type: 'date' },
      'user_name': { type: 'keyword' }
    }
  }
});
*/