Added cryptocompare impl + file renamer
This commit is contained in:
parent
50b70cf6fb
commit
a7d41ca69b
|
@ -0,0 +1,20 @@
|
||||||
|
import fs from 'fs-extra'
|
||||||
|
|
||||||
|
let folder = './data/CCCAGG/BTC/USD/'
|
||||||
|
|
||||||
|
const read = file=> fs.readFile( folder+file, 'utf8' )
|
||||||
|
//.then( getTimeline )
|
||||||
|
const parseCSV = data=> data.split('\n').map( s=> s.split(',') )
|
||||||
|
const getTime = values=> values[0]
|
||||||
|
|
||||||
|
fs.readdir( folder )
|
||||||
|
.then( files=> files.map( file=> {
|
||||||
|
let tl = fs.readFile( folder+file, 'utf8' )
|
||||||
|
.then( parseCSV )
|
||||||
|
.then( csv=> csv[csv.length-1][0] )
|
||||||
|
.then( last=>
|
||||||
|
fs.move( folder+file, folder+file.replace(/\.csv$/, '-'+last+'.csv') )
|
||||||
|
)
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
|
@ -1,7 +1,12 @@
|
||||||
import fetch from 'node-fetch'
|
import config from '../config'
|
||||||
|
import fetch from 'fetch-filecache-for-crawling'
|
||||||
|
//fetch.setParameter( 'logToConsole', true )
|
||||||
|
fetch.setParameter( 'cacheFolder', `${config.dataDir}/fetched/` )
|
||||||
|
global.fetch = fetch
|
||||||
|
|
||||||
import cc from 'cryptocompare'
|
import cc from 'cryptocompare'
|
||||||
import fs from 'fs-extra'
|
import fs from 'fs-extra'
|
||||||
global.fetch = fetch
|
import { log } from '../utils/log'
|
||||||
|
|
||||||
cc.rateLimit = function( apiKey )
|
cc.rateLimit = function( apiKey )
|
||||||
{
|
{
|
||||||
|
@ -24,8 +29,58 @@ export function loadByPath( ccfs )
|
||||||
const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 }
|
const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 }
|
||||||
const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 }
|
const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 }
|
||||||
|
|
||||||
|
// esm providers/cryptocompare:coins BTC ETH
|
||||||
|
// coins([ 'BTC', 'ETH' ]).then
|
||||||
|
export async function coins( syms )
|
||||||
|
{
|
||||||
|
let list = await cc.coinList()
|
||||||
|
log('args')(syms)
|
||||||
|
return Promise.all(
|
||||||
|
Object.keys( list.Data )
|
||||||
|
.filter( sym=> syms.length? ~syms.indexOf(sym): true )
|
||||||
|
.map( sym=> {
|
||||||
|
fs.outputFile(`${config.dataDir}/coins/${list.Data[sym].Symbol}/index.json`, JSON.stringify(list.Data[sym]) )
|
||||||
|
return fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'})
|
||||||
|
.then( res=> res.text() )
|
||||||
|
.then(
|
||||||
|
html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
|
||||||
|
)
|
||||||
|
.then( json=> (fs.outputFile(`${config.dataDir}/coins/${sym}/overview.json`, json), json) )
|
||||||
|
.then(
|
||||||
|
str=> JSON.parse( str )
|
||||||
|
)
|
||||||
|
|
||||||
|
})
|
||||||
|
/*
|
||||||
|
// .map( log() )
|
||||||
|
.map( coin=> (fs.outputFile(`${config.dataDir}/coins/${list.Data[coin].Symbol}/index.json`, JSON.stringify(list.Data[coin])),coin) )
|
||||||
|
.filter( sym=> syms.length? ~syms.indexOf(sym): true )
|
||||||
|
.map( log('selected') )
|
||||||
|
.map( sym=> fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) )
|
||||||
|
.map( prom=> prom.then( res=> res.text() ) )
|
||||||
|
.map( prom=> prom.then(
|
||||||
|
html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.map( prom=> prom.then( coin=>(fs.outputFile(`${config.dataDir}/coins/${coin.Data.General.Symbol}/overview.json`,coin),coin) ) )
|
||||||
|
.map( prom=> prom.then(
|
||||||
|
html=> JSON.parse( html )
|
||||||
|
)
|
||||||
|
// .then(log('ress'))
|
||||||
|
// .then(coin=>fs.outputFile(`${config.dataDir}/coins/${coin.Symbol}/infos.json`,coin))
|
||||||
|
)
|
||||||
|
/*
|
||||||
|
.map( html=> ({
|
||||||
|
name: html.match(//)[1]
|
||||||
|
, startDate: html.match(//)[1]
|
||||||
|
}))
|
||||||
|
*/
|
||||||
|
).then( ress=> ress.reduce((all,json)=>({...all,[json.Data.General.Symbol]:json}), {}) )
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
export {
|
export {
|
||||||
coinList as coins
|
coinList as coinsL
|
||||||
, exchangeList as exchanges
|
, exchangeList as exchanges
|
||||||
} from 'cryptocompare'
|
} from 'cryptocompare'
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
import fetch from 'node-fetch'
|
||||||
|
import config from '../config'
|
||||||
|
|
||||||
|
global.fetch = (...args)=> fetch( ...args )
|
||||||
|
.then( res=> (res.text().then( txt=> fs.outputFile(`${config.dataDir}/fetched/${Date.now()}`)))
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
export const log = (...args)=> obj=> ( console.log( ...args, obj ), obj )
|
Loading…
Reference in New Issue