Added cryptocompare impl + file renamer

This commit is contained in:
dig 2021-08-30 19:28:23 +02:00
parent 50b70cf6fb
commit a7d41ca69b
4 changed files with 86 additions and 3 deletions

20
ccrename.js Normal file
View File

@ -0,0 +1,20 @@
import fs from 'fs-extra'
let folder = './data/CCCAGG/BTC/USD/'
const read = file=> fs.readFile( folder+file, 'utf8' )
//.then( getTimeline )
const parseCSV = data=> data.split('\n').map( s=> s.split(',') )
const getTime = values=> values[0]
fs.readdir( folder )
.then( files=> files.map( file=> {
let tl = fs.readFile( folder+file, 'utf8' )
.then( parseCSV )
.then( csv=> csv[csv.length-1][0] )
.then( last=>
fs.move( folder+file, folder+file.replace(/\.csv$/, '-'+last+'.csv') )
)
})
)

View File

@ -1,7 +1,12 @@
import fetch from 'node-fetch'
import config from '../config'
import fetch from 'fetch-filecache-for-crawling'
//fetch.setParameter( 'logToConsole', true )
fetch.setParameter( 'cacheFolder', `${config.dataDir}/fetched/` )
global.fetch = fetch
import cc from 'cryptocompare'
import fs from 'fs-extra'
global.fetch = fetch
import { log } from '../utils/log'
cc.rateLimit = function( apiKey )
{
@ -24,8 +29,58 @@ export function loadByPath( ccfs )
const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 }
const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 }
// esm providers/cryptocompare:coins BTC ETH
// coins([ 'BTC', 'ETH' ]).then
export async function coins( syms )
{
let list = await cc.coinList()
log('args')(syms)
return Promise.all(
Object.keys( list.Data )
.filter( sym=> syms.length? ~syms.indexOf(sym): true )
.map( sym=> {
fs.outputFile(`${config.dataDir}/coins/${list.Data[sym].Symbol}/index.json`, JSON.stringify(list.Data[sym]) )
return fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'})
.then( res=> res.text() )
.then(
html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
)
.then( json=> (fs.outputFile(`${config.dataDir}/coins/${sym}/overview.json`, json), json) )
.then(
str=> JSON.parse( str )
)
})
/*
// .map( log() )
.map( coin=> (fs.outputFile(`${config.dataDir}/coins/${list.Data[coin].Symbol}/index.json`, JSON.stringify(list.Data[coin])),coin) )
.filter( sym=> syms.length? ~syms.indexOf(sym): true )
.map( log('selected') )
.map( sym=> fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) )
.map( prom=> prom.then( res=> res.text() ) )
.map( prom=> prom.then(
html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
)
)
.map( prom=> prom.then( coin=>(fs.outputFile(`${config.dataDir}/coins/${coin.Data.General.Symbol}/overview.json`,coin),coin) ) )
.map( prom=> prom.then(
html=> JSON.parse( html )
)
// .then(log('ress'))
// .then(coin=>fs.outputFile(`${config.dataDir}/coins/${coin.Symbol}/infos.json`,coin))
)
/*
.map( html=> ({
name: html.match(//)[1]
, startDate: html.match(//)[1]
}))
*/
).then( ress=> ress.reduce((all,json)=>({...all,[json.Data.General.Symbol]:json}), {}) )
}
export {
coinList as coins
coinList as coinsL
, exchangeList as exchanges
} from 'cryptocompare'

7
utils/fetchWithCache.js Normal file
View File

@ -0,0 +1,7 @@
import fetch from 'node-fetch'
import config from '../config'
global.fetch = (...args)=> fetch( ...args )
.then( res=> (res.text().then( txt=> fs.outputFile(`${config.dataDir}/fetched/${Date.now()}`)))

1
utils/log.js Normal file
View File

@ -0,0 +1 @@
export const log = (...args)=> obj=> ( console.log( ...args, obj ), obj )