Merge pull request #97 from 0xProject/feat/scrape-otc-event

Feat/scrape otc event

Author: Ktl-XV

.dockerignore

@@ -1 +1,2 @@
 postgres/
+node_modules

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
     postgres:
-        image: postgres:9.6
+        image: postgres:10.14
         environment:
             - POSTGRES_USER=api
             - POSTGRES_PASSWORD=api

event-pipeline/Dockerfile

@@ -10,7 +10,7 @@ RUN apk update && \
     apk upgrade && \
     apk add --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \
     yarn --frozen-lockfile --no-cache
-    
+
 RUN yarn build
 
 # Stage 2

event-pipeline/migrations/1641418834000-AddOtcOrderFilledEvents.ts

@@ -0,0 +1,53 @@
+import { MigrationInterface, QueryRunner, Table } from 'typeorm';
+
+const eventsOtcOrderFilledEvents = new Table({
+    name: 'events.otc_order_filled_events',
+    columns: [
+        { name: 'observed_timestamp', type: 'bigint' },
+        { name: 'contract_address', type: 'varchar' },
+        { name: 'transaction_hash', type: 'varchar', isPrimary: true },
+        { name: 'transaction_index', type: 'bigint' },
+        { name: 'log_index', type: 'bigint', isPrimary: true },
+        { name: 'block_hash', type: 'varchar' },
+        { name: 'block_number', type: 'bigint' },
+
+        { name: 'order_hash', type: 'varchar' },
+        { name: 'maker_address', type: 'varchar' },
+        { name: 'taker_address', type: 'varchar' },
+        { name: 'maker_token_address', type: 'varchar' },
+        { name: 'taker_token_address', type: 'varchar' },
+        { name: 'maker_token_filled_amount', type: 'numeric' },
+        { name: 'taker_token_filled_amount', type: 'numeric' },
+    ],
+});
+
+const indexQuery = `
+    CREATE INDEX otc_order_filled_events_transaction_hash_index
+        ON events.otc_order_filled_events (transaction_hash);
+    CREATE INDEX otc_order_filled_events_block_number_index
+        ON events.otc_order_filled_events (block_number);
+    CREATE INDEX otc_order_filled_events_maker_address_index
+        ON events.otc_order_filled_events (maker_address);
+    CREATE INDEX otc_order_filled_events_order_hash_index
+        ON events.otc_order_filled_events (order_hash);
+
+`;
+
+const dropIndexQuery = `
+    DROP INDEX events.otc_order_filled_events_transaction_hash_index;
+    DROP INDEX events.otc_order_filled_events_block_number_index;
+    DROP INDEX events.otc_order_filled_events_maker_address_index;
+    DROP INDEX events.otc_order_filled_events_order_hash_index;
+`;
+
+export class AddOtcOrderFilledEvents1641418834000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.createTable(eventsOtcOrderFilledEvents);
+        await queryRunner.query(indexQuery);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(dropIndexQuery);
+        await queryRunner.dropTable(eventsOtcOrderFilledEvents);
+    }
+}

event-pipeline/src/constants.ts

@@ -38,6 +38,8 @@ export const SWAP_V3_EVENT_TOPIC = [
     '0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67',
     '0x000000000000000000000000def1c0ded9bec7f1a1670819833240f027b25eff',
 ];
+export const OTC_ORDER_FILLED_EVENT_TOPIC = ['0xac75f773e3a92f1a02b12134d65e1f47f8a14eabe4eaf1e24624918e6a8b269f'];
+export const OTC_ORDERS_FEATURE_START_BLOCK = 13143075;
 
 export {
     EXPIRED_RFQ_ORDER_ABI,
@@ -309,3 +311,53 @@ export const V3_FILL_ABI = {
     name: 'Fill',
     type: 'event',
 };
+
+export const OTC_ORDER_FILLED_ABI = {
+    anonymous: false,
+    inputs: [
+        {
+            indexed: false,
+            internalType: 'bytes32',
+            name: 'orderHash',
+            type: 'bytes32',
+        },
+        {
+            indexed: false,
+            internalType: 'address',
+            name: 'maker',
+            type: 'address',
+        },
+        {
+            indexed: false,
+            internalType: 'address',
+            name: 'taker',
+            type: 'address',
+        },
+        {
+            indexed: false,
+            internalType: 'address',
+            name: 'makerToken',
+            type: 'address',
+        },
+        {
+            indexed: false,
+            internalType: 'address',
+            name: 'takerToken',
+            type: 'address',
+        },
+        {
+            indexed: false,
+            internalType: 'uint128',
+            name: 'makerTokenFilledAmount',
+            type: 'uint128',
+        },
+        {
+            indexed: false,
+            internalType: 'uint128',
+            name: 'takerTokenFilledAmount',
+            type: 'uint128',
+        },
+    ],
+    name: 'OtcOrderFilled',
+    type: 'event',
+};

event-pipeline/src/entities/index.ts

@@ -32,6 +32,7 @@ export { EpochFinalizedEvent } from './epoch_finalized_event';
 export { RewardsPaidEvent } from './rewards_paid_event';
 export { StakingProxyDeployment } from './staking_proxy_deployment';
 export { TransactionExecutionEvent } from './transaction_execution_event';
+export { OtcOrderFilledEvent } from './otc_order_filled_event';
 
 @Entity({ name: 'blocks', schema: 'events' })
 export class Block extends BlockTemplate {}

event-pipeline/src/entities/otc_order_filled_event.ts

@@ -0,0 +1,34 @@
+import { bigNumberTransformer } from '@0x/pipeline-utils';
+import { BigNumber } from '@0x/utils';
+import { Column, Entity } from 'typeorm';
+
+import { Event } from '@0x/pipeline-utils';
+
+import { ProxyType } from '../utils';
+
+// These events come directly from the Exchange contract and are fired whenever
+// someone fills an order.
+@Entity({ name: 'otc_order_filled_events', schema: 'events' })
+export class OtcOrderFilledEvent extends Event {
+    // The order hash.
+    @Column({ name: 'order_hash' })
+    public orderHash!: string;
+    // The address of the maker.
+    @Column({ name: 'maker_address' })
+    public makerAddress!: string;
+    // The address of the taker (may be null).
+    @Column({ name: 'taker_address' })
+    public takerAddress!: string;
+    // The address of the maker token.
+    @Column({ name: 'maker_token_address', type: 'varchar', nullable: true })
+    public makerTokenAddress!: string | null;
+    // The address of the taker token.
+    @Column({ name: 'taker_token_address', type: 'varchar', nullable: true })
+    public takerTokenAddress!: string | null;
+    // The amount of the maker asset which was filled.
+    @Column({ name: 'maker_token_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public makerTokenFilledAmount!: BigNumber;
+    // The amount of the taker asset which was filled.
+    @Column({ name: 'taker_token_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public takerTokenFilledAmount!: BigNumber;
+}

event-pipeline/src/ormconfig.ts

@@ -25,6 +25,7 @@ import {
     EpochFinalizedEvent,
     RewardsPaidEvent,
     StakingProxyDeployment,
+    OtcOrderFilledEvent,
     ERC20BridgeTransferEvent,
     TransformedERC20Event,
     TransactionExecutionEvent,
@@ -58,6 +59,7 @@ const entities = [
     EpochFinalizedEvent,
     RewardsPaidEvent,
     StakingProxyDeployment,
+    OtcOrderFilledEvent,
     ERC20BridgeTransferEvent,
     TransformedERC20Event,
     TransactionExecutionEvent,

event-pipeline/src/parsers/events/otc_order_filled_events.ts

@@ -0,0 +1,37 @@
+import { assetDataUtils } from '@0x/order-utils';
+import { AssetProxyId } from '@0x/types';
+import { RawLogEntry } from 'ethereum-types';
+
+import { OtcOrderFilledEvent } from '../../entities';
+import { convertAssetProxyIdToType } from '../../utils';
+import { parseEvent } from './parse_event';
+import { OTC_ORDER_FILLED_ABI } from '../../constants';
+
+const abiCoder = require('web3-eth-abi');
+
+/**
+ * Parses raw event logs for a OTC Fill Event and returns an array of
+ * OtcOrderFilledEvent entities.
+ * @param eventLog Raw event log (e.g. returned from contract-wrappers).
+ */
+
+export function parseOtcOrderFilledEvent(eventLog: RawLogEntry): OtcOrderFilledEvent {
+    const otcOrderFillEvent = new OtcOrderFilledEvent();
+    parseEvent(eventLog, otcOrderFillEvent);
+
+    const decodedLog = abiCoder.decodeLog(
+        OTC_ORDER_FILLED_ABI.inputs,
+        eventLog.data,
+        eventLog.topics.slice(1, eventLog.topics.length),
+    );
+
+    otcOrderFillEvent.orderHash = decodedLog.orderHash.toLowerCase();
+    otcOrderFillEvent.makerAddress = decodedLog.maker.toLowerCase();
+    otcOrderFillEvent.takerAddress = decodedLog.taker.toLowerCase();
+    otcOrderFillEvent.makerTokenAddress = decodedLog.makerToken.toLowerCase();
+    otcOrderFillEvent.takerTokenAddress = decodedLog.takerToken.toLowerCase();
+    otcOrderFillEvent.makerTokenFilledAmount = decodedLog.makerTokenFilledAmount;
+    otcOrderFillEvent.takerTokenFilledAmount = decodedLog.takerTokenFilledAmount;
+
+    return otcOrderFillEvent;
+}

event-pipeline/src/scripts/pull_and_save_events_by_topic.ts

@@ -14,6 +14,7 @@ import {
     FillEvent,
     V4CancelEvent,
     ExpiredRfqOrderEvent,
+    OtcOrderFilledEvent,
 } from '../entities';
 
 import { ETHEREUM_RPC_URL, FIRST_SEARCH_BLOCK } from '../config';
@@ -36,6 +37,8 @@ import {
     NEW_BRIDGEFILL_BLOCK,
     FLASHWALLET_ADDRESS,
     SWAP_V3_EVENT_TOPIC,
+    OTC_ORDER_FILLED_EVENT_TOPIC,
+    OTC_ORDERS_FEATURE_START_BLOCK,
 } from '../constants';
 
 import { parseTransformedERC20Event } from '../parsers/events/transformed_erc20_events';
@@ -54,6 +57,7 @@ import { parseFillEvent } from '../parsers/events/fill_events';
 import { parseNativeFillFromFillEvent } from '../parsers/events/fill_events';
 import { parseV4CancelEvent } from '../parsers/events/v4_cancel_events';
 import { parseExpiredRfqOrderEvent } from '../parsers/events/expired_rfq_order_events';
+import { parseOtcOrderFilledEvent } from '../parsers/events/otc_order_filled_events';
 
 import { PullAndSaveEventsByTopic } from './utils/event_abi_utils';
 
@@ -216,6 +220,18 @@ export class EventsByTopicScraper {
                 parseExpiredRfqOrderEvent,
                 {},
             ),
+            pullAndSaveEventsByTopic.getParseSaveEventsByTopic<OtcOrderFilledEvent>(
+                connection,
+                web3Source,
+                latestBlockWithOffset,
+                'OtcOrderFilledEvent',
+                'otc_order_filled_events',
+                OTC_ORDER_FILLED_EVENT_TOPIC,
+                EXCHANGE_PROXY_ADDRESS,
+                OTC_ORDERS_FEATURE_START_BLOCK,
+                parseOtcOrderFilledEvent,
+                {},
+            ),
         ]);
 
         const endTime = new Date().getTime();