diff --git a/Gemfile.lock b/Gemfile.lock index 9d475a7..cc9c041 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -189,9 +189,8 @@ GEM mail (2.5.4) mime-types (~> 1.16) treetop (~> 1.4.8) - mediawiki-gateway (0.5.2) + mediawiki-gateway (0.6.1) activesupport - mediawiki-gateway rest-client (>= 1.3.0) meta_search (1.1.3) actionpack (~> 3.1) @@ -212,6 +211,7 @@ GEM net-ssh (2.7.0) net-ssh-gateway (1.2.0) net-ssh (>= 2.6.5) + netrc (0.7.7) newrelic_rpm (3.6.8.168) nokogiri (1.6.2.1) mini_portile (= 0.6.0) @@ -275,8 +275,9 @@ GEM ref (1.0.5) responders (1.0.0) railties (>= 3.2, < 5) - rest-client (1.6.7) - mime-types (>= 1.16) + rest-client (1.7.2) + mime-types (>= 1.16, < 3.0) + netrc (~> 0.7) rsolr (1.0.10) builder (>= 2.1.2) rspec (3.0.0) diff --git a/lib/tasks/update_snpedia_marshall_array.rake b/lib/tasks/update_snpedia_marshall_array.rake new file mode 100644 index 0000000..b560e03 --- /dev/null +++ b/lib/tasks/update_snpedia_marshall_array.rake @@ -0,0 +1,22 @@ +require 'media_wiki' + +namespace :snpedia do + desc "update snpedia array" + task :download => :environment do + file = Rails::root.join('marshalled_snpedia_array') + + old = Marshal.load(File.open(file)) + puts "There are #{old.length} SNPs in the old array" + + mw = MediaWiki::Gateway.new("http://bots.snpedia.com/api.php") + puts 'Downloading all SNPs' + new = mw.category_members('Category:Is_a_snp') + new.map! { |x| x.downcase } + + puts "There are #{new.length} SNPs in the new array, dumping now" + File.open(file, 'wb') do |f| + f.write Marshal.dump(new) + end + puts "Done!" + end +end diff --git a/marshalled_snpedia_array b/marshalled_snpedia_array index e073b75..c982221 100644 Binary files a/marshalled_snpedia_array and b/marshalled_snpedia_array differ diff --git a/test/data/vcr_cassettes/snpedia_worker.yml b/test/data/vcr_cassettes/snpedia_worker.yml index 694fbb4..7becd4c 100644 --- a/test/data/vcr_cassettes/snpedia_worker.yml +++ b/test/data/vcr_cassettes/snpedia_worker.yml @@ -8,20 +8,20 @@ http_interactions: string: '' headers: Accept: - - '*/*; q=0.5, application/xml' + - "*/*; q=0.5, application/xml" Accept-Encoding: - gzip User-Agent: - - MediaWiki::Gateway/0.5.2 + - MediaWiki::Gateway/0.6.1 response: status: code: 200 message: OK headers: Server: - - Apache/2.2.25 (Amazon) + - Apache/2.2.27 (Amazon) X-Powered-By: - - PHP/5.3.27 + - PHP/5.3.28 X-Content-Type-Options: - nosniff X-Frame-Options: @@ -33,19 +33,23 @@ http_interactions: Content-Type: - text/xml; charset=utf-8 Content-Length: - - '2331' + - '2533' Accept-Ranges: - bytes Date: - - Sat, 14 Dec 2013 14:45:09 GMT + - Thu, 28 Aug 2014 06:26:52 GMT X-Varnish: - - 2007274517 2007273299 + - 2090605832 2090605709 Age: - - '420' + - '49' + X-Cache: + - MISS from proxy10.uq.edu.au + X-Cache-Lookup: + - MISS from proxy10.uq.edu.au:80 Via: - - 1.1 varnish + - 1.1 varnish, 1.1 proxy10.uq.edu.au:80 (squid/2.7.STABLE9) Connection: - - keep-alive + - close body: encoding: UTF-8 string: Form talkConceptConcept + talkModuleModule talk http_version: - recorded_at: Sat, 14 Dec 2013 14:45:09 GMT + recorded_at: Thu, 28 Aug 2014 06:26:46 GMT - request: method: get - uri: http://www.snpedia.com/api.php?action=query&apfrom=&aplimit=500&apnamespace=0&apprefix=rs12979860(&format=xml&list=allpages&maxlag=5 + uri: http://www.snpedia.com/api.php?action=query&aplimit=500&apnamespace=0&apprefix=rs12979860(&format=xml&list=allpages&maxlag=5 body: encoding: US-ASCII string: '' headers: Accept: - - '*/*; q=0.5, application/xml' + - "*/*; q=0.5, application/xml" Accept-Encoding: - gzip User-Agent: - - MediaWiki::Gateway/0.5.2 + - MediaWiki::Gateway/0.6.1 response: status: code: 200 message: OK headers: Server: - - Apache/2.2.25 (Amazon) + - Apache/2.2.27 (Amazon) X-Powered-By: - - PHP/5.3.27 + - PHP/5.3.28 X-Content-Type-Options: - nosniff X-Frame-Options: @@ -114,22 +120,26 @@ http_interactions: Accept-Ranges: - bytes Date: - - Sat, 14 Dec 2013 14:45:10 GMT + - Thu, 28 Aug 2014 06:26:52 GMT X-Varnish: - - 2007274518 2007273300 + - '2090605833' Age: - - '420' + - '0' + X-Cache: + - MISS from proxy10.uq.edu.au + X-Cache-Lookup: + - MISS from proxy10.uq.edu.au:80 Via: - - 1.1 varnish + - 1.1 varnish, 1.1 proxy10.uq.edu.au:80 (squid/2.7.STABLE9) Connection: - - keep-alive + - close body: encoding: UTF-8 string:

http_version: - recorded_at: Sat, 14 Dec 2013 14:45:10 GMT + recorded_at: Thu, 28 Aug 2014 06:26:47 GMT - request: method: get uri: http://www.snpedia.com/api.php?action=query&format=xml&maxlag=5&prop=revisions&rvlimit=1&rvprop=ids&titles=Rs12979860(C%3BC) @@ -138,20 +148,20 @@ http_interactions: string: '' headers: Accept: - - '*/*; q=0.5, application/xml' + - "*/*; q=0.5, application/xml" Accept-Encoding: - gzip User-Agent: - - MediaWiki::Gateway/0.5.2 + - MediaWiki::Gateway/0.6.1 response: status: code: 200 message: OK headers: Server: - - Apache/2.2.25 (Amazon) + - Apache/2.2.27 (Amazon) X-Powered-By: - - PHP/5.3.27 + - PHP/5.3.28 X-Content-Type-Options: - nosniff X-Frame-Options: @@ -167,22 +177,26 @@ http_interactions: Accept-Ranges: - bytes Date: - - Sat, 14 Dec 2013 14:45:10 GMT + - Thu, 28 Aug 2014 06:26:53 GMT X-Varnish: - - 2007274521 2007273302 + - '2090605841' Age: - - '420' + - '0' + X-Cache: + - MISS from proxy10.uq.edu.au + X-Cache-Lookup: + - MISS from proxy10.uq.edu.au:80 Via: - - 1.1 varnish + - 1.1 varnish, 1.1 proxy10.uq.edu.au:80 (squid/2.7.STABLE9) Connection: - - keep-alive + - close body: encoding: UTF-8 string: http_version: - recorded_at: Sat, 14 Dec 2013 14:45:10 GMT + recorded_at: Thu, 28 Aug 2014 06:26:47 GMT - request: method: get uri: http://www.snpedia.com/api.php?action=query&format=xml&maxlag=5&prop=revisions&rvprop=content&titles=Rs12979860(C%3BC) @@ -191,20 +205,20 @@ http_interactions: string: '' headers: Accept: - - '*/*; q=0.5, application/xml' + - "*/*; q=0.5, application/xml" Accept-Encoding: - gzip User-Agent: - - MediaWiki::Gateway/0.5.2 + - MediaWiki::Gateway/0.6.1 response: status: code: 200 message: OK headers: Server: - - Apache/2.2.25 (Amazon) + - Apache/2.2.27 (Amazon) X-Powered-By: - - PHP/5.3.27 + - PHP/5.3.28 X-Content-Type-Options: - nosniff X-Frame-Options: @@ -220,15 +234,19 @@ http_interactions: Accept-Ranges: - bytes Date: - - Sat, 14 Dec 2013 14:45:10 GMT + - Thu, 28 Aug 2014 06:26:53 GMT X-Varnish: - - 2007274523 2007273306 + - '2090605844' Age: - - '420' + - '0' + X-Cache: + - MISS from proxy10.uq.edu.au + X-Cache-Lookup: + - MISS from proxy10.uq.edu.au:80 Via: - - 1.1 varnish + - 1.1 varnish, 1.1 proxy10.uq.edu.au:80 (squid/2.7.STABLE9) Connection: - - keep-alive + - close body: encoding: UTF-8 string: |- @@ -241,7 +259,7 @@ http_interactions: |summary=~80% of such hepatitis C patients respond to treatment }} http_version: - recorded_at: Sat, 14 Dec 2013 14:45:10 GMT + recorded_at: Thu, 28 Aug 2014 06:26:47 GMT - request: method: get uri: http://www.snpedia.com/api.php?action=query&format=xml&maxlag=5&prop=revisions&rvlimit=1&rvprop=ids&titles=Rs12979860(C%3BT) @@ -250,20 +268,20 @@ http_interactions: string: '' headers: Accept: - - '*/*; q=0.5, application/xml' + - "*/*; q=0.5, application/xml" Accept-Encoding: - gzip User-Agent: - - MediaWiki::Gateway/0.5.2 + - MediaWiki::Gateway/0.6.1 response: status: code: 200 message: OK headers: Server: - - Apache/2.2.25 (Amazon) + - Apache/2.2.27 (Amazon) X-Powered-By: - - PHP/5.3.27 + - PHP/5.3.28 X-Content-Type-Options: - nosniff X-Frame-Options: @@ -279,22 +297,26 @@ http_interactions: Accept-Ranges: - bytes Date: - - Sat, 14 Dec 2013 14:45:10 GMT + - Thu, 28 Aug 2014 06:26:53 GMT X-Varnish: - - '2007274524' + - '2090605847' Age: - '0' + X-Cache: + - MISS from proxy10.uq.edu.au + X-Cache-Lookup: + - MISS from proxy10.uq.edu.au:80 Via: - - 1.1 varnish + - 1.1 varnish, 1.1 proxy10.uq.edu.au:80 (squid/2.7.STABLE9) Connection: - - keep-alive + - close body: encoding: UTF-8 string: http_version: - recorded_at: Sat, 14 Dec 2013 14:45:11 GMT + recorded_at: Thu, 28 Aug 2014 06:26:48 GMT - request: method: get uri: http://www.snpedia.com/api.php?action=query&format=xml&maxlag=5&prop=revisions&rvprop=content&titles=Rs12979860(C%3BT) @@ -303,20 +325,20 @@ http_interactions: string: '' headers: Accept: - - '*/*; q=0.5, application/xml' + - "*/*; q=0.5, application/xml" Accept-Encoding: - gzip User-Agent: - - MediaWiki::Gateway/0.5.2 + - MediaWiki::Gateway/0.6.1 response: status: code: 200 message: OK headers: Server: - - Apache/2.2.25 (Amazon) + - Apache/2.2.27 (Amazon) X-Powered-By: - - PHP/5.3.27 + - PHP/5.3.28 X-Content-Type-Options: - nosniff X-Frame-Options: @@ -332,15 +354,19 @@ http_interactions: Accept-Ranges: - bytes Date: - - Sat, 14 Dec 2013 14:45:11 GMT + - Thu, 28 Aug 2014 06:26:54 GMT X-Varnish: - - '2007274528' + - '2090605848' Age: - '0' + X-Cache: + - MISS from proxy10.uq.edu.au + X-Cache-Lookup: + - MISS from proxy10.uq.edu.au:80 Via: - - 1.1 varnish + - 1.1 varnish, 1.1 proxy10.uq.edu.au:80 (squid/2.7.STABLE9) Connection: - - keep-alive + - close body: encoding: UTF-8 string: |- @@ -353,7 +379,7 @@ http_interactions: |summary=~20-40% of such hepatitis C patients respond to treatment }} http_version: - recorded_at: Sat, 14 Dec 2013 14:45:11 GMT + recorded_at: Thu, 28 Aug 2014 06:26:48 GMT - request: method: get uri: http://www.snpedia.com/api.php?action=query&format=xml&maxlag=5&prop=revisions&rvlimit=1&rvprop=ids&titles=Rs12979860(T%3BT) @@ -362,20 +388,20 @@ http_interactions: string: '' headers: Accept: - - '*/*; q=0.5, application/xml' + - "*/*; q=0.5, application/xml" Accept-Encoding: - gzip User-Agent: - - MediaWiki::Gateway/0.5.2 + - MediaWiki::Gateway/0.6.1 response: status: code: 200 message: OK headers: Server: - - Apache/2.2.25 (Amazon) + - Apache/2.2.27 (Amazon) X-Powered-By: - - PHP/5.3.27 + - PHP/5.3.28 X-Content-Type-Options: - nosniff X-Frame-Options: @@ -391,22 +417,26 @@ http_interactions: Accept-Ranges: - bytes Date: - - Sat, 14 Dec 2013 14:45:11 GMT + - Thu, 28 Aug 2014 06:26:54 GMT X-Varnish: - - '2007274533' + - '2090605851' Age: - '0' + X-Cache: + - MISS from proxy10.uq.edu.au + X-Cache-Lookup: + - MISS from proxy10.uq.edu.au:80 Via: - - 1.1 varnish + - 1.1 varnish, 1.1 proxy10.uq.edu.au:80 (squid/2.7.STABLE9) Connection: - - keep-alive + - close body: encoding: UTF-8 string: http_version: - recorded_at: Sat, 14 Dec 2013 14:45:11 GMT + recorded_at: Thu, 28 Aug 2014 06:26:48 GMT - request: method: get uri: http://www.snpedia.com/api.php?action=query&format=xml&maxlag=5&prop=revisions&rvprop=content&titles=Rs12979860(T%3BT) @@ -415,20 +445,20 @@ http_interactions: string: '' headers: Accept: - - '*/*; q=0.5, application/xml' + - "*/*; q=0.5, application/xml" Accept-Encoding: - gzip User-Agent: - - MediaWiki::Gateway/0.5.2 + - MediaWiki::Gateway/0.6.1 response: status: code: 200 message: OK headers: Server: - - Apache/2.2.25 (Amazon) + - Apache/2.2.27 (Amazon) X-Powered-By: - - PHP/5.3.27 + - PHP/5.3.28 X-Content-Type-Options: - nosniff X-Frame-Options: @@ -444,15 +474,19 @@ http_interactions: Accept-Ranges: - bytes Date: - - Sat, 14 Dec 2013 14:45:11 GMT + - Thu, 28 Aug 2014 06:26:54 GMT X-Varnish: - - '2007274535' + - '2090605853' Age: - '0' + X-Cache: + - MISS from proxy10.uq.edu.au + X-Cache-Lookup: + - MISS from proxy10.uq.edu.au:80 Via: - - 1.1 varnish + - 1.1 varnish, 1.1 proxy10.uq.edu.au:80 (squid/2.7.STABLE9) Connection: - - keep-alive + - close body: encoding: UTF-8 string: |- @@ -465,5 +499,5 @@ http_interactions: |summary=~20-25% of such hepatitis C patients respond to treatment }} http_version: - recorded_at: Sat, 14 Dec 2013 14:45:12 GMT + recorded_at: Thu, 28 Aug 2014 06:26:49 GMT recorded_with: VCR 2.8.0