Skip to content
This repository was archived by the owner on Nov 10, 2021. It is now read-only.

Throttling logic #15

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Throttling logic #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
faae0e2
- add throttling
anonymoussprocket Mar 20, 2021
ab0cbfb
Merge branch 'master' into throttle
anonymoussprocket Mar 20, 2021
8cc5ee2
Merge branch 'master' into throttle
anonymoussprocket Mar 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .serverless/cloudformation-template-update-stack.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@
"S3Bucket": {
"Ref": "ServerlessDeploymentBucket"
},
"S3Key": "serverless/hicetnunc-apiv2/dev/1616132797292-2021-03-19T05:46:37.292Z/hicetnunc-apiv2.zip"
"S3Key": "serverless/hicetnunc-apiv2/dev/1616180496084-2021-03-19T19:01:36.084Z/hicetnunc-apiv2.zip"
},
"FunctionName": "hicetnunc-apiv2-dev-handler",
"Handler": "index.handler",
Expand All @@ -170,14 +170,14 @@
"HandlerLogGroup"
]
},
"HandlerLambdaVersionl43NXxTCsqpysRMWyXSvU0lsarIv6k41EVMGBVAAAI": {
"HandlerLambdaVersionxKXiD47XxbV8a0CrQwjITk3MONXyb5JqjEgIyp7EM": {
"Type": "AWS::Lambda::Version",
"DeletionPolicy": "Retain",
"Properties": {
"FunctionName": {
"Ref": "HandlerLambdaFunction"
},
"CodeSha256": "yP30UhwdABdpM1UtDvnZ21GW5zg9jII05figW7uS74k="
"CodeSha256": "nF8jqb9R6Jwn5BvBH8TIt8EhoKsJFfZyPkF8eSd4WH4="
}
},
"ApiGatewayRestApi": {
Expand Down Expand Up @@ -389,7 +389,7 @@
"MethodResponses": []
}
},
"ApiGatewayDeployment1616132791720": {
"ApiGatewayDeployment1616180485677": {
"Type": "AWS::ApiGateway::Deployment",
"Properties": {
"RestApiId": {
Expand Down Expand Up @@ -451,7 +451,7 @@
"HandlerLambdaFunctionQualifiedArn": {
"Description": "Current Lambda function version",
"Value": {
"Ref": "HandlerLambdaVersionl43NXxTCsqpysRMWyXSvU0lsarIv6k41EVMGBVAAAI"
"Ref": "HandlerLambdaVersionxKXiD47XxbV8a0CrQwjITk3MONXyb5JqjEgIyp7EM"
}
},
"ServiceEndpoint": {
Expand Down
Binary file modified .serverless/hicetnunc-apiv2.zip
View file
Open in desktop
Binary file not shown.
16 changes: 8 additions & 8 deletions .serverless/serverless-state.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@
"S3Bucket": {
"Ref": "ServerlessDeploymentBucket"
},
"S3Key": "serverless/hicetnunc-apiv2/dev/1616132797292-2021-03-19T05:46:37.292Z/hicetnunc-apiv2.zip"
"S3Key": "serverless/hicetnunc-apiv2/dev/1616180496084-2021-03-19T19:01:36.084Z/hicetnunc-apiv2.zip"
},
"FunctionName": "hicetnunc-apiv2-dev-handler",
"Handler": "index.handler",
Expand All @@ -187,14 +187,14 @@
"HandlerLogGroup"
]
},
"HandlerLambdaVersionl43NXxTCsqpysRMWyXSvU0lsarIv6k41EVMGBVAAAI": {
"HandlerLambdaVersionxKXiD47XxbV8a0CrQwjITk3MONXyb5JqjEgIyp7EM": {
"Type": "AWS::Lambda::Version",
"DeletionPolicy": "Retain",
"Properties": {
"FunctionName": {
"Ref": "HandlerLambdaFunction"
},
"CodeSha256": "yP30UhwdABdpM1UtDvnZ21GW5zg9jII05figW7uS74k="
"CodeSha256": "nF8jqb9R6Jwn5BvBH8TIt8EhoKsJFfZyPkF8eSd4WH4="
}
},
"ApiGatewayRestApi": {
Expand Down Expand Up @@ -406,7 +406,7 @@
"MethodResponses": []
}
},
"ApiGatewayDeployment1616132791720": {
"ApiGatewayDeployment1616180485677": {
"Type": "AWS::ApiGateway::Deployment",
"Properties": {
"RestApiId": {
Expand Down Expand Up @@ -468,7 +468,7 @@
"HandlerLambdaFunctionQualifiedArn": {
"Description": "Current Lambda function version",
"Value": {
"Ref": "HandlerLambdaVersionl43NXxTCsqpysRMWyXSvU0lsarIv6k41EVMGBVAAAI"
"Ref": "HandlerLambdaVersionxKXiD47XxbV8a0CrQwjITk3MONXyb5JqjEgIyp7EM"
}
},
"ServiceEndpoint": {
Expand Down Expand Up @@ -627,7 +627,7 @@
"memory": 1024,
"runtime": "nodejs12.x",
"vpc": {},
"versionLogicalId": "HandlerLambdaVersionl43NXxTCsqpysRMWyXSvU0lsarIv6k41EVMGBVAAAI"
"versionLogicalId": "HandlerLambdaVersionxKXiD47XxbV8a0CrQwjITk3MONXyb5JqjEgIyp7EM"
}
},
"configValidationMode": "warn",
Expand Down Expand Up @@ -2557,7 +2557,7 @@
"node_modules/@types/aws-lambda/trigger/sqs.d.ts"
],
"artifact": "/home/crzy/Desktop/programs/lambdas-aws/api-final/new2/hicetnunc-api/.serverless/hicetnunc-apiv2.zip",
"artifactDirectoryName": "serverless/hicetnunc-apiv2/dev/1616132797292-2021-03-19T05:46:37.292Z"
"artifactDirectoryName": "serverless/hicetnunc-apiv2/dev/1616180496084-2021-03-19T19:01:36.084Z"
},
"plugins": {
"$ref": "$[\"service\"][\"plugins\"]"
Expand All @@ -2574,7 +2574,7 @@
"artifact": "/home/crzy/Desktop/programs/lambdas-aws/api-final/new2/hicetnunc-api/.serverless/hicetnunc-apiv2.zip"
},
"package": {
"artifactDirectoryName": "serverless/hicetnunc-apiv2/dev/1616132797292-2021-03-19T05:46:37.292Z",
"artifactDirectoryName": "serverless/hicetnunc-apiv2/dev/1616180496084-2021-03-19T19:01:36.084Z",
"artifact": "hicetnunc-apiv2.zip"
}
}
43 changes: 38 additions & 5 deletions conseilUtil.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ const conseiljs = require('conseiljs')
const fetch = require('node-fetch')
const log = require('loglevel')
const BigNumber = require('bignumber.js')
const pThrottle = require('p-throttle')
// const { performance } = require('perf_hooks'); // NOTE: disabled for prod

const logger = log.getLogger('conseiljs')
logger.setLevel('error', false)
Expand All @@ -11,8 +13,26 @@ const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech'
const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud
const tezosNode = ''

const throttleConseil = pThrottle({ limit: 15, interval: 1200 })

const mainnet = require('./config').networkConfig


const hDAOFeed = async () => {

let hDAOQuery = conseiljs.ConseilQueryBuilder.blankQuery();
hDAOQuery = conseiljs.ConseilQueryBuilder.addFields(hDAOQuery, 'key', 'value');
hDAOQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDAOQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.curationsPtr])
hDAOQuery = conseiljs.ConseilQueryBuilder.setLimit(hDAOQuery, 10_000)

let hDAOResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDAOQuery);
return hDAOResult.map(e => {
return {
token_id : parseInt(e.key),
hDAO_balance : parseInt((e.value).split(' ')[1])
}
})
}
/**
* Returns a list of nft token ids and amounts that a given address owns.
*
Expand Down Expand Up @@ -212,11 +232,11 @@ const getArtisticUniverse = async (max_time) => {
let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery();
mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash');
mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction'])
mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.BETWEEN, [1612240919000, max_time]) // after 2021 Feb 1
mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [1612240919000]) // after 2021 Feb 1
mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied'])
mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol])
mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT'])
mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 20_000)
mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 30_000)

const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData(
{ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' },
Expand Down Expand Up @@ -275,23 +295,29 @@ const getArtisticUniverse = async (max_time) => {

const objectQueries = queryChunks.map(c => makeObjectQuery(c))

// const a = performance.now()
let universe = []
await Promise.all(
objectQueries.map(async (q) => {
const r = []
let r = []
try {
r = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q)
r = await throttleConseilQuery('big_map_contents', q)
.then(result => result.map(row => {
const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1')
const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1')
const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7);

universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []})
})) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected
} catch (error) {
console.log('failed at query', q, 'with error', error)
} finally {
return r
}}))

// const b = performance.now()
// console.log(`time ${b - a}`)

return universe
}

Expand Down Expand Up @@ -353,10 +379,17 @@ const chunkArray = (arr, len) => { // TODO: move to util.js
return chunks;
}

const throttleConseilQuery = throttleConseil(async (table, query) => {
const result = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', table, query)

return Promise.resolve(result)
})

module.exports = {
getCollectionForAddress,
gethDaoBalanceForAddress,
getArtisticOutputForAddress,
getObjectById,
getArtisticUniverse
getArtisticUniverse,
hDAOFeed
}
Loading