Usage

by LazyImport

main.ts

import etl from '@jrmc/etl'

const UserSource = () => import('./user_array_source.js')
const UserTransform = () => import('./user_array_to_db_transform.js')
const UserDestination = () => import('./user_db_destination.js')

await etl.run({
  source: UserSource,
  transform: UserTransform,
  destination: UserDestination,
})

user_array_source.ts

import { Source } from '@jrmc/etl/types'

export default class UserArraySource implements Source {
  async *each() {
    const dataArray = [
      { lastname: 'Doe', firstname: 'John', age: 30 },
      { lastname: 'Doe', firstname: 'Jane', age: 25 },
    ]

    for (let item of dataArray) {
      yield item
    }
  }
}

user_array_to_db_transform.ts

import { Transform } from '@jrmc/etl/types'

type User = {
  firstname: string
  lastname: string
  age: number
}

export default class UserArrayToDbTransform implements Transform {
  async process(row: User) {
    return {
      name: `${row.firstname} ${row.lastname}`,
      age: row.age,
    }
  }
}

user_db_destination.ts

import { Destination } from '@jrmc/etl/types'

export default class UserDbDestination implements Destination {
  async write(row: any) {
    User.create(row)
  }
}

by LazyImport with options

main.ts

import etl from '@jrmc/etl'

const UserSource = () => import('./user_array_source.js')
const UserDestination = () => import('./user_db_destination.js')

await etl.run({
  source: [UserSource, { 
    data: [
      { lastname: 'Doe', firstname: 'John', age: 30 },
      { lastname: 'Doe', firstname: 'Jane', age: 25 },
    ]
  }],
  destination: [UserDestination, { 
    age: 22
  }],
})

user_array_source.ts

import { Source } from '@jrmc/etl/types'

type Options = Record<string, Array<Object>>

export default class TestWithOptionsSource implements Source {
  #data: Array<Object>

  constructor(options: Options) {
    this.#data = options.data || [];
  }

  async *each() {
    for (let item of this.#data) {
      yield item
    }
  }
}

user_db_destination.ts

import { Destination } from '@jrmc/etl/types'

type Options = {
  age: number
}

type Person = {
  firstname: string
  lastname: string
  age: number
}

export default class TestWithOptionsDestination implements Destination {
  #age: number | null

  constructor(options: Options) {
    this.#age = options.age || null;
  }

  async write(row: Person) {
    User.create({ lastname: row.lastname, firstname: row.firstname, age: this.#age ? this.#age : row.age })
  }
}

by Functions

Use AsyncIterator and AsyncWithData functions

import etl from '@jrmc/etl'

await etl.run({
  source: async function* () {
    const dataArray = [
      { lastname: 'Doe', firstname: 'John', age: 30 },
      { lastname: 'Doe', firstname: 'Jane', age: 25 },
    ]

    for (let item of dataArray) {
      yield item
    }
  },
  transform: async function (row: any) {
    return {
      name: `${row.firstname} ${row.lastname}`,
      age: row.age,
    }
  },
  destination: async function (row: any) {
    User.create(row)
  },
})

get results

run method return array if Destination class return a result.

main.ts

import etl from '@jrmc/etl'

const UserSource = () => import('./user_array_source.js')
const UserTransform = () => import('./user_array_to_db_transform.js')
const UserDestination = () => import('./user_db_destination.js')

const results = await etl.run({
  source: UserSource,
  transform: UserTransform,
  destination: UserDestination,
})

/*
return :
[
  { name: 'John Doe', age: 30 },
  { name: 'Jane Doe', age: 25 },
]
*/

user_array_source.ts

import { Source } from '@jrmc/etl/types'

export default class UserArraySource implements Source {
  async *each() {
    const dataArray = [
      { lastname: 'Doe', firstname: 'John', age: 30 },
      { lastname: 'Doe', firstname: 'Jane', age: 25 },
    ]

    for (let item of dataArray) {
      yield item
    }
  }
}

user_array_to_db_transform.ts

import { Transform } from '@jrmc/etl/types'

type User = {
  firstname: string
  lastname: string
  age: number
}

export default class UserArrayToDbTransform implements Transform {
  async process(row: User) {
    return {
      name: `${row.firstname} ${row.lastname}`,
      age: row.age,
    }
  }
}

user_db_destination.ts

import { Destination } from '@jrmc/etl/types'

export default class UserDbDestination implements Destination {
  async write(row: any) {
    return row
  }
}