From 20160c3296852b8f4d43beb34ae404045a0d2bea Mon Sep 17 00:00:00 2001 From: wang--ge Date: Mon, 18 Sep 2023 18:12:40 +0800 Subject: [PATCH] Support different data sources for unite_cur due to disappearance of Yahoo feed --- 0001-units-2.17-units_cur-validate.patch | 441 ++++++++++++++++++++--- units.spec | 5 +- 2 files changed, 385 insertions(+), 61 deletions(-) diff --git a/0001-units-2.17-units_cur-validate.patch b/0001-units-2.17-units_cur-validate.patch index 237903e..2f99064 100644 --- a/0001-units-2.17-units_cur-validate.patch +++ b/0001-units-2.17-units_cur-validate.patch @@ -1,23 +1,28 @@ -From 9d1129f41f193a47d6791f44f14abe9479999266 Mon Sep 17 00:00:00 2001 -From: Kamil Dudka -Date: Wed, 8 Aug 2018 17:42:17 +0200 -Subject: [PATCH] units_cur: validate rate data from server +From 321eba0f7943a7172dbfce6d470145e48ada6397 Mon Sep 17 00:00:00 2001 +From: wang--ge +Date: Mon, 18 Sep 2023 17:23:17 +0800 +Subject: [PATCH] units 2.17 units_cur validate --- - units_cur | 72 ++++++++++++++++++++++++++++++++++++++++++------------- - 1 file changed, 55 insertions(+), 17 deletions(-) + units_cur | 322 +++++++++++++++++++++++++++++++++++++++++------------- + 1 file changed, 245 insertions(+), 77 deletions(-) diff --git a/units_cur b/units_cur -index 00281d8..d625570 100755 +index 00281d8..70c8d8e 100755 --- a/units_cur +++ b/units_cur -@@ -28,8 +28,12 @@ from __future__ import absolute_import, division, print_function +@@ -28,8 +28,17 @@ from __future__ import absolute_import, division, print_function # # -version = '4.2' -+version = '4.3' ++version = '5.0' ++# Version 5.0: ++# ++# Rewrite to support multiple different data sources due to disappearance ++# of the Yahoo feed. Includes support for base currency selection. ++# +# Version 4.3: 20 July 2018 +# +# Validate rate data from server @@ -25,22 +30,124 @@ index 00281d8..d625570 100755 # Version 4.2: 18 April 2018 # # Handle case of empty/malformed entry returned from the server -@@ -55,6 +59,10 @@ from sys import exit, stderr, stdout +@@ -55,10 +64,17 @@ from sys import exit, stderr, stdout outfile_name = 'currency.units' +# valid metals + +validmetals = ['silver','gold','platinum'] ++ ++PRIMITIVE = '! # Base unit, the primitive unit of currency' + # This exchange rate table lists the currency ISO 4217 codes, their # long text names, and any fixed definitions. If the definition is # empty then units_cur will query the server for a value. -@@ -271,11 +279,19 @@ ap.add_argument('-v','--verbose', - help='display details when fetching currency data', - ) -+ ++rate_index = 1 + currency = OrderedDict([ + ('ATS', ['austriaschilling', '1|13.7603 euro']), + ('BEF', ['belgiumfranc', '1|40.3399 euro']), +@@ -83,13 +99,14 @@ currency = OrderedDict([ + ('BGN', ['bulgarialev', '1|1.9558 euro']), + ('BAM', ['bosniaconvertiblemark','germanymark']), + ('KMF', ['comorosfranc', '1|491.96775 euro']), +- ('XOF', ['westafricanfranc', '1|655.957 euro']), ++ ('XOF', ['westafricafranc', '1|655.957 euro']), + ('XPF', ['cfpfranc', '1|119.33 euro']), +- ('XAF', ['centralafricancfafranc','1|655.957 euro']), ++ ('XAF', ['centralafricacfafranc','1|655.957 euro']), + ('AED', ['uaedirham','']), + ('AFN', ['afghanafghani','']), + ('ALL', ['albanialek','']), + ('AMD', ['armeniadram','']), ++ ('ANG', ['antillesguilder','']), + ('AOA', ['angolakwanza','']), + ('ARS', ['argentinapeso','']), + ('AUD', ['australiadollar','']), +@@ -109,7 +126,7 @@ currency = OrderedDict([ + ('BTN', ['bhutanngultrum','']), + ('BWP', ['botswanapula','']), + ('BYN', ['belarusruble','']), +- ('BYR', ['oldbelarusruble','10000 BYN']), ++ ('BYR', ['oldbelarusruble','1|10000 BYN']), + ('BZD', ['belizedollar','']), + ('CAD', ['canadadollar','']), + ('CDF', ['drcfranccongolais','']), +@@ -127,7 +144,7 @@ currency = OrderedDict([ + ('DZD', ['algeriadinar','']), + ('EGP', ['egyptpound','']), + ('ERN', ['eritreanakfa','']), +- ('ETB', ['ethiopianbirr','']), ++ ('ETB', ['ethiopiabirr','']), + ('EUR', ['euro','']), + ('FJD', ['fijidollar','']), + ('FKP', ['falklandislandspound','']), +@@ -164,10 +181,9 @@ currency = OrderedDict([ + ('KZT', ['kazakhstantenge','']), + ('LAK', ['laokip','']), + ('LBP', ['lebanonpound','']), +- ('LKR', ['srilankanrupee','']), ++ ('LKR', ['srilankarupee','']), + ('LRD', ['liberiadollar','']), +- ('LTL', ['lithuanialita','']), +- ('LVL', ['latvialat','']), ++ ('LSL', ['lesotholoti','']), + ('LYD', ['libyadinar','']), + ('MAD', ['moroccodirham','']), + ('MDL', ['moldovaleu','']), +@@ -176,13 +192,14 @@ currency = OrderedDict([ + ('MMK', ['myanmarkyat','']), + ('MNT', ['mongoliatugrik','']), + ('MOP', ['macaupataca','']), +- ('MRO', ['mauritaniaouguiya','']), ++ ('MRO', ['mauritaniaoldouguiya','1|10 MRU']), ++ ('MRU', ['mauritaniaouguiya', '']), + ('MUR', ['mauritiusrupee','']), + ('MVR', ['maldiverufiyaa','']), + ('MWK', ['malawikwacha','']), + ('MXN', ['mexicopeso','']), + ('MYR', ['malaysiaringgit','']), +- ('MZN', ['mozambicanmetical','']), ++ ('MZN', ['mozambiquemetical','']), + ('NAD', ['namibiadollar','']), + ('NGN', ['nigerianaira','']), + ('NIO', ['nicaraguacordobaoro','']), +@@ -212,7 +229,9 @@ currency = OrderedDict([ + ('SLL', ['sierraleoneleone','']), + ('SOS', ['somaliaschilling','']), + ('SRD', ['surinamedollar','']), +- ('STD', ['saotome&principedobra','']), ++ ('SSP', ['southsudanpound','']), ++ ('STD', ['saotome&principeolddobra','']), ++ ('STN', ['saotome&principedobra','']), + ('SVC', ['elsalvadorcolon','']), + ('SYP', ['syriapound','']), + ('SZL', ['swazilandlilangeni','']), +@@ -227,15 +246,15 @@ currency = OrderedDict([ + ('TZS', ['tanzaniashilling','']), + ('UAH', ['ukrainehryvnia','']), + ('UGX', ['ugandaschilling','']), +- ('USD', ['unitedstatesdollar', 'US$']), ++ ('USD', ['US$', '']), + ('UYU', ['uruguaypeso','']), + ('UZS', ['uzbekistansum','']), +- ('VEF', ['venezuelabolivar','']), +- ('VEB', ['venezuelaoldbolivar', '1000 VEF']), ++ ('VEF', ['venezuelabolivarfuerte','']), ++ ('VES', ['venezuelabolivarsoberano','']), + ('VND', ['vietnamdong','']), + ('VUV', ['vanuatuvatu','']), + ('WST', ['samoatala','']), +- ('XAF', ['centralafricancfafranc','']), ++ ('XAF', ['centralafricacfafranc','']), + ('XCD', ['eastcaribbeandollar','']), + ('XDR', ['specialdrawingrights','']), + ('YER', ['yemenrial','']), +@@ -244,6 +263,147 @@ currency = OrderedDict([ + ('ZWL', ['zimbabwedollar','']), + ]) + +def validfloat(x): + try: + float(x) @@ -48,56 +155,246 @@ index 00281d8..d625570 100755 + except ValueError: + return False + - outfile_name = ap.parse_args().output_file - verbose = ap.parse_args().verbose ++def addrate(verbose,form,code,rate): ++ if code not in currency.keys(): ++ if (verbose): ++ stderr.write('Got unknown currency with code {}\n'.format(code)) ++ else: ++ if not currency[code][rate_index]: ++ if validfloat(rate): ++ currency[code][rate_index] = form.format(rate) ++ else: ++ stderr.write('Got invalid rate "{}" for currency "{}"\n'.format( ++ rate, code)) ++ elif verbose: ++ if currency[code][rate_index] != form.format(rate): ++ stderr.write('Got value "{}" for currency "{}" but ' ++ 'it is already defined as {}\n'.format(rate, code, ++ currency[code][rate_index])) ++ ++def getjson(address,args=None): ++ try: ++ res = requests.get(address,args) ++ res.raise_for_status() ++ return(res.json()) ++ except requests.exceptions.RequestException as e: ++ stderr.write('Error connecting to currency server:\n{}.\n'.format(e)) ++ exit(1) ++ ++######################################################## ++# ++# Connect to floatrates for currency update ++# ++ ++def floatrates(verbose,base,dummy): ++ webdata = getjson('https://www.floatrates.com/daily/'+base+'.json') ++ for index in webdata: ++ entry = webdata[index] ++ if 'rate' not in entry or 'code' not in entry: # Skip empty/bad entries ++ if verbose: ++ stderr.write('Got bad entry from server: '+str(entry)+'\n') ++ else: ++ addrate(verbose,'{} '+base,entry['code'],entry['inverseRate']) ++ currency[base][rate_index] = PRIMITIVE ++ return('FloatRates ('+base+' base)') ++ ++######################################################## ++# ++# Connect to European central bank site ++# ++ ++def eubankrates(verbose,base,dummy): ++ if verbose and base!='EUR': ++ stderr.write('European bank uses euro for base currency. Specified base {} ignored.\n'.format(base)) ++ import xml.etree.ElementTree as ET ++ try: ++ res=requests.get('https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml') ++ res.raise_for_status() ++ data = ET.fromstring(res.content)[2][0] ++ except requests.exceptions.RequestException as e: ++ stderr.write('Error connecting to currency server:\n{}.\n'. ++ format(e)) ++ exit(1) ++ for entry in data.iter(): ++ if entry.get('time'): ++ continue ++ rate = entry.get('rate') ++ code = entry.get('currency') ++ if not rate or not code: # Skip empty/bad entries ++ if verbose: ++ stderr.write('Got bad entry from server, code {} and rate {}\n'.format(code,rate)) ++ else: ++ addrate(verbose,'1|{} euro', code, rate) ++ currency['EUR'][rate_index]=PRIMITIVE ++ return('the European Central Bank (euro base)') ++ ++######################################################## ++# ++# Connect to fixer.io (requires API key) ++# ++# Free API key does not allow changing base currency ++# With free key only euro base is supported, and https is not allowed ++# ++ ++def fixer(verbose,base,key): ++ if not key: ++ stderr.write('API key required for this source\n') ++ exit(1) ++ if verbose and base!='EUR': ++ stderr.write('Fixer uses euro for base currency. Specified base {} ignored.\n'.format(base)) ++ webdata = getjson('http://data.fixer.io/api/latest', {'access_key':key}) ++ if not webdata['success']: ++ stderr.write('Currency server error: '+webdata['error']['info']) ++ exit(1) ++ for code in webdata['rates']: ++ addrate(verbose,'1|{} euro', code, webdata['rates'][code]) ++ currency['EUR'][rate_index] = PRIMITIVE ++ return('Fixer (euro base)') ++ ++######################################################## ++# ++# Connect to openexchangerates (requires API key) ++# ++# Free API key does not allow changing the base currency ++# ++ ++def openexchangerates(verbose,base,key): ++ if not key: ++ stderr.write('API key required for this source\n') ++ exit(1) ++ if verbose and base!='USD': ++ stderr.write('Open Exchange Rates uses US dollar for base currency. Specified base {} ignored.\n'.format(base)) ++ webdata = getjson('https://openexchangerates.org/api/latest.json', ++ {'app_id':key} ++ ) ++ for code in webdata['rates']: ++ addrate(verbose,'1|{} US$', code, webdata['rates'][code]) ++ currency['USD'][rate_index] = PRIMITIVE ++ return('open exchange rates (USD base)') ++ ++####################################################### ++# ++# list of valid source names and corresponding functions ++# ++ ++sources = { ++ 'floatrates': floatrates, ++ 'eubank' : eubankrates, ++ 'fixer' : fixer, ++ 'openexchangerates': openexchangerates, ++} ++ ++####################################################### ++# ++# Argument Processing ++# ++ + ap = ArgumentParser( + description="Update currency information for 'units' " + "into the specified filename or if no filename is " +@@ -271,64 +431,43 @@ ap.add_argument('-v','--verbose', + help='display details when fetching currency data', + ) - try: +-outfile_name = ap.parse_args().output_file +-verbose = ap.parse_args().verbose ++ap.add_argument('-s','--source',choices=list(sources.keys()), ++ default='floatrates', ++ help='set currency data source', ++) + +-try: - res = requests.get('http://finance.yahoo.com/webservice/v1/symbols' -+ res = requests.get('https://finance.yahoo.com/webservice/v1/symbols' - '/allcurrencies/quote?format=json') - res.raise_for_status() - webdata = res.json()['list']['resources'] -@@ -299,10 +315,16 @@ for data in webdata: - stderr.write('Got unknown currency with code {}\n'.format(code)) - else: - if not currency[code][rate_index]: +- '/allcurrencies/quote?format=json') +- res.raise_for_status() +- webdata = res.json()['list']['resources'] +-except requests.exceptions.RequestException as e: +- stderr.write('Error connecting to currency server:\n{}.\n'. +- format(e)) ++ap.add_argument('-b','--base',default='USD', ++ help='set the base currency (when allowed by source). BASE should be a 3 letter ISO currency code, e.g. USD. The specified currency will be the primitive currency unit used by units. Only the floatrates source supports this option.', ++) ++ ++ap.add_argument('-k','--key',default='', ++ help='set API key for sources that require it' ++) ++ ++args = ap.parse_args() ++outfile_name = args.output_file ++verbose = args.verbose ++source = args.source ++base = args.base ++apikey = args.key ++ ++if base not in currency.keys(): ++ stderr.write('Base currency {} is not a known currency code.\n'.format(base)) + exit(1) +- + +-rate_index = 1 +-for data in webdata: +- entry = data['resource']['fields'] +- if 'price' not in entry or 'symbol' not in entry: # Skip empty/bad entries +- if verbose: +- stderr.write('Got bad entry from server: '+str(entry)+'\n') +- else: +- rate = entry['price'] +- code = entry['symbol'][0:3] +- if code not in currency.keys(): +- if (verbose): +- stderr.write('Got unknown currency with code {}\n'.format(code)) +- else: +- if not currency[code][rate_index]: - currency[code][rate_index] = '1|{} US$'.format(rate) -+ if validfloat(rate): -+ currency[code][rate_index] = '1|{} US$'.format(rate) -+ else: -+ stderr.write('Got invalid rate "{}" for currency "{}"\n'.format( -+ rate, code)) - elif verbose: +- elif verbose: - stderr.write('Got value "{}" for currency "{}" but ' - 'it is already defined\n'.format(rate, code)) -+ if currency[code][rate_index] != '1|{} US$'.format(rate): -+ stderr.write('Got value "{}" for currency "{}" but ' -+ 'it is already defined as {}\n'.format(rate, code, -+ currency[code][rate_index])) ++######################################################## ++# ++# Fetch currency data from specified curerncy source ++# - +- ++sourcename = sources[source](verbose,base,apikey) ++ # Delete currencies where we have no rate data -@@ -313,17 +335,15 @@ for code in currency.keys(): +-for code in currency.keys(): ++for code in list(currency.keys()): + if not currency[code][rate_index]: + if verbose: +- stderr.write('No data for {}'.format(code)) ++ stderr.write('No data for {}\n'.format(code)) del currency[code] - - try: +- +-try: - req = requests.get('http://services.packetizer.com/spotprices/?f=json') -+ req = requests.get('https://services.packetizer.com/spotprices/?f=json') - req.raise_for_status() - metals = req.json() - except requests.exceptions.RequestException as e: - stderr.write('Error connecting to spotprices server:\n{}\n'.format(e)) - exit(1) - +- req.raise_for_status() +- metals = req.json() +-except requests.exceptions.RequestException as e: +- stderr.write('Error connecting to spotprices server:\n{}\n'.format(e)) +- exit(1) +- -del metals['date'] - - try: +-try: - req = requests.get('http://services.packetizer.com/btc/?f=json') -+ req = requests.get('https://services.packetizer.com/btc/?f=json') - req.raise_for_status() - bitcoin = req.json() - except requests.exceptions.RequestException as e: -@@ -344,13 +364,31 @@ ratestr = '\n'.join( +- req.raise_for_status() +- bitcoin = req.json() +-except requests.exceptions.RequestException as e: +- stderr.write('Error connecting to bitcoin server:\n{}\n'.format(e)) +- exit(1) + + cnames = [currency[code][0] for code in currency.keys()] + crates = [currency[code][1] for code in currency.keys()] +@@ -336,40 +475,69 @@ crates = [currency[code][1] for code in currency.keys()] + codestr = '\n'.join('{:23}{}'. + format(code, name) for (code,name) in zip(currency.keys(), cnames)) + +-datestr = date.today().isoformat() +- + maxlen = max(len(name) for name in cnames) + 2 + + ratestr = '\n'.join( '{:{}}{}'.format(name, maxlen, rate) for (name, rate) in zip(cnames, crates) ) @@ -105,8 +402,15 @@ index 00281d8..d625570 100755 - metal + 'price', - price, - ) for metal, price in metals.items()) -- ++####################################################### ++# ++# Get precious metals data and bitcoin ++# + -bitcoinstr = '{:{}}{} US$ # From services.packetizer.com/btc\n'.format( ++metals = getjson('https://services.packetizer.com/spotprices',{'f':'json'}) ++bitcoin = getjson('https://services.packetizer.com/btc',{'f':'json'}) ++ +metallist = ['']*len(validmetals) +for metal, price in metals.items(): + if metal in validmetals: @@ -119,10 +423,9 @@ index 00281d8..d625570 100755 + stderr.write('Got value "{}" for metal "{}" but ' + 'it is already defined\n'.format(price,metal)) + else: -+ stderr.write('Got invalid rate "{}" for metal "{}"\n'.format( -+ price, metal)) ++ stderr.write('Got invalid rate "{}" for metal "{}"\n'.format(price,metal)) + elif metal != 'date' and verbose: # Don't print a message for the "date" entry -+ stderr.write('Got unknown metal "{}" with value "{}"\n',metal,price) ++ stderr.write('Got unknown metal "{}" with value "{}"\n'.format(metal,price)) +metalstr = '\n'.join(metallist) + +if validfloat(bitcoin['usd']): @@ -131,11 +434,27 @@ index 00281d8..d625570 100755 +else: + stderr.write('Got invalid bitcoin rate "{}"\n', bitcoint['usd']) + bitcointstr='' -+ ++ ++####################################################### ++# ++# Format output and write the currency file ++# ++datestr = date.today().isoformat() ++ outstr = ( """# ISO Currency Codes -@@ -366,9 +404,9 @@ outstr = ( + + {codestr} + +-# Currency exchange rates from Yahoo Finance (finance.yahoo.com) ++# Currency exchange rates source + +-!message Currency exchange rates from finance.yahoo.com on {datestr} ++!message Currency exchange rates from {sourcename} on {datestr} + + {ratestr} + {bitcoinstr} # Precious metals prices from Packetizer (services.packetizer.com/spotprices) @@ -143,10 +462,12 @@ index 00281d8..d625570 100755 +{metalstr} -""".format(codestr=codestr, datestr=datestr, ratestr=ratestr, ozzystr=ozzystr, +- bitcoinstr=bitcoinstr) +""".format(codestr=codestr, datestr=datestr, ratestr=ratestr, metalstr=metalstr, - bitcoinstr=bitcoinstr) ++ bitcoinstr=bitcoinstr, sourcename=sourcename) ).replace('\n', linesep) + try: -- -2.17.1 +2.33.0 diff --git a/units.spec b/units.spec index abbdefb..b446e90 100644 --- a/units.spec +++ b/units.spec @@ -1,6 +1,6 @@ Name: units Version: 2.17 -Release: 9 +Release: 10 Summary: A utility for converting amounts from one unit to another License: GPLv3+ URL: https://www.gnu.org/software/units/units.html @@ -68,6 +68,9 @@ fi %{_mandir}/man1/* %changelog +* Mon Sep 18 2023 Ge Wang - 2.17-10 +- Support different data sources for units_cur due to disappearance of Yahoo feed. + * Thu Sep 07 2023 Ge Wang - 2.17-9 - Add install requirement