import config from '../config' import fetch from 'fetch-filecache-for-crawling' //fetch.setParameter( 'logToConsole', true ) fetch.setParameter( 'cacheFolder', `${config.dataDir}/fetched/` ) global.fetch = fetch import cc from 'cryptocompare' import fs from 'fs-extra' import { log } from '../utils/log' cc.rateLimit = function( apiKey ) { return fetch( `https://min-api.cryptocompare.com/stats/rate/limit${apiKey ? `?api_key=${apiKey}` : ''}` ) .then( res=> { if( !res.ok ){ throw new Error(`${res.status} ${res.statusText}`) }else return res.json() }) .then( body=> { if( body.Response === 'Error' ){ throw body.Message }else return body }) } // loadByPath( "EXCHANGE/SYM/SYM/PERIOD/TS|/TS|LIMIT" ).then() export function loadByPath( ccfs ) { let [ exchange, fsym, tsym, period, start, limit = 2000 ] = ccfs.split("/") , [ , aggregate, tUnit ] = period.match( /(\d+)([HDWM]*)/ ) , loader = 'histo' + (tUnit == '' ? 'Minute' : tUnit == 'H' ? 'Hour' : 'Day') // calculate toTs from limit and start console.log( loader, aggregate*timeFrames[tUnit] ) return cc[loader]( fsym, tsym, { allData: true, limit: 2000, aggregate: aggregate*timeFrames[tUnit], exchange } ) } const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 } const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 } // esm providers/cryptocompare:coins BTC ETH // coins([ 'BTC', 'ETH' ]).then export async function coins( syms ) { let list = await cc.coinList() log('args')(syms) return Promise.all( Object.keys( list.Data ) .filter( sym=> syms.length? ~syms.indexOf(sym): true ) .map( sym=> { fs.outputFile(`${config.dataDir}/coins/${list.Data[sym].Symbol}/index.json`, JSON.stringify(list.Data[sym]) ) return fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) .then( res=> res.text() ) .then( html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1] ) .then( json=> (fs.outputFile(`${config.dataDir}/coins/${sym}/overview.json`, json), json) ) .then( str=> JSON.parse( str ) ) }) /* // .map( log() ) .map( coin=> (fs.outputFile(`${config.dataDir}/coins/${list.Data[coin].Symbol}/index.json`, JSON.stringify(list.Data[coin])),coin) ) .filter( sym=> syms.length? ~syms.indexOf(sym): true ) .map( log('selected') ) .map( sym=> fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) ) .map( prom=> prom.then( res=> res.text() ) ) .map( prom=> prom.then( html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1] ) ) .map( prom=> prom.then( coin=>(fs.outputFile(`${config.dataDir}/coins/${coin.Data.General.Symbol}/overview.json`,coin),coin) ) ) .map( prom=> prom.then( html=> JSON.parse( html ) ) // .then(log('ress')) // .then(coin=>fs.outputFile(`${config.dataDir}/coins/${coin.Symbol}/infos.json`,coin)) ) /* .map( html=> ({ name: html.match(//)[1] , startDate: html.match(//)[1] })) */ ).then( ress=> ress.reduce((all,json)=>({...all,[json.Data.General.Symbol]:json}), {}) ) } export { coinList as coinsL , exchangeList as exchanges } from 'cryptocompare'