Data dumps/Status format
We are now producing a json-formatted status file for the dump of each wiki on a particular date. It is updated as files are produced. Example format (indentation added for readability):
{"version": "0.8",
"jobs": {
"articlesmultistreamdump": {
"status": "done",
"files": {
"elwiki-20170401-pages-articles-multistream.xml.bz2":
{"url": "/elwiki/20170401/elwiki-20170401-pages-articles-multistream.xml.bz2", "size": 271090061},
"elwiki-20170401-pages-articles-multistream-index.txt.bz2":
{"url": "/elwiki/20170401/elwiki-20170401-pages-articles-multistream-index.txt.bz2", "size": 2717257}
},
"updated": "2017-04-03 14:31:20"},
"langlinkstable": {
"status": "done",
"files": {"elwiki-20170401-langlinks.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-langlinks.sql.gz",
"sha1": "6816faf2ddbe69fe92ecb7c71f061a1f18054d92",
"md5": "58c448d467801247ddcf4b2d0eddbee8",
"size": 44516915}
},
"updated": "2017-04-02 09:15:14"
},
"pagelinkstable": {
"status": "done",
"files": {"elwiki-20170401-pagelinks.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-pagelinks.sql.gz",
"sha1": "272291561fb44d3b23427d455897d861c3d2cf1b",
"md5": "fc08a8a9e1914604f8370148e24d7c5d",
"size": 46811358}
},
"updated": "2017-04-02 09:18:53"},
"imagetable": {
"status": "done",
"files": {"elwiki-20170401-image.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-image.sql.gz",
"sha1": "0e1f299481842c9bb296c6adc8e072b7aa0d2b79",
"md5": "2ff70134e9a963aebbd90eb1a560dd54",
"size": 2216907}
},
"updated": "2017-04-02 09:15:19"
},
"iwlinkstable": {
"status": "done",
"files": {"elwiki-20170401-iwlinks.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-iwlinks.sql.gz",
"sha1": "f96864af070dfb7cfcf27223425c120e9a7a0943",
"md5": "6d3c6d016553058eee814621d3211b3c",
"size": 2204376}
},
"updated": "2017-04-02 09:16:31"
},
"usergroupstable": {
"status": "done",
"files": {"elwiki-20170401-user_groups.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-user_groups.sql.gz",
"sha1": "d99b7a23891602ad323b8e8bc93a08f675213dbc",
"md5": "277e05192dff59eaa4e6e4008cd810be",
"size": 1291}
},
"updated": "2017-04-02 09:15:47"
},
"xmlpagelogsdump": {
"status": "done",
"files": {"elwiki-20170401-pages-logging.xml.gz": {
"url": "/elwiki/20170401/elwiki-20170401-pages-logging.xml.gz",
"sha1": "257e62ddd008639edc69ab71449db28e081833df",
"md5": "c7307dd58b297d6220ba75342dbc8d53",
"size": 26669476}
},
"updated": "2017-04-05 16:05:30"
},
"changetagstable": {
"status": "done",
"files": {"elwiki-20170401-change_tag.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-change_tag.sql.gz",
"sha1": "99e2321039f38db91eb29e99ca08d78b443ab94a",
"md5": "0e2a0d38516e4f7090705b0bf42c6582",
"size": 2264729}
},
"updated": "2017-04-02 09:17:21"
},
"imagelinkstable": {
"status": "done",
"files": {"elwiki-20170401-imagelinks.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-imagelinks.sql.gz",
"sha1": "8f8f2d264e97290b6dd523f0db2aa3c3ed91d458",
"md5": "f1720958a5a2bcff022da971d4d0d95b",
"size": 5331794}
},
"updated": "2017-04-02 09:15:43"
},
"pagetable": {
"status": "done",
"files": {"elwiki-20170401-page.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-page.sql.gz",
"sha1": "1dbea87e06ee22f989ef5c73365ed8a228c7d948",
"md5": "d398b194eb5dc0cba604408b324f33d4",
"size": 15013830}
},
"updated": "2017-04-02 09:15:33"
},
"xmlflowhistorydump": {
"status": "done",
"files": {"elwiki-20170401-flowhistory.xml.bz2": {
"url": "/elwiki/20170401/elwiki-20170401-flowhistory.xml.bz2",
"sha1": "4670f375e65dbee01cd1ea98168bd3754d088ff4",
"md5": "8ba8baf2b2b0b218eec6fc255dc2d19d",
"size": 309637}
},
"updated": "2017-04-05 16:06:20"
},
"metacurrentdump": {
"status": "done",
"files": {"elwiki-20170401-pages-meta-current.xml.bz2": {
"url": "/elwiki/20170401/elwiki-20170401-pages-meta-current.xml.bz2",
"sha1": "676756c1750e9643ba444d775493806c4e27ac4b",
"md5": "e290fc403eb5f9818167ef2fbeeaac2e",
"size": 326017055}
},
"updated": "2017-04-04 16:14:40"
},
"pagepropstable": {
"status": "done",
"files": {"elwiki-20170401-page_props.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-page_props.sql.gz",
"sha1": "93bac8fc0968f2d200481e6ad8d3ef68dcaad403",
"md5": "66e613d39c13aabd3e0803502348a039",
"size": 5085458}
},
"updated": "2017-04-02 09:17:27"
},
"metahistory7zdump": {
"status": "done",
"files": {"elwiki-20170401-pages-meta-history.xml.7z": {
"url": "/elwiki/20170401/elwiki-20170401-pages-meta-history.xml.7z",
"sha1": "ec7c2a4e61258b1d6e8d5924ac01261dc64af4ff",
"md5": "43420adca10298f272ab5cd1977eb812",
"size": 1102932726}
},
"updated": "2017-04-06 10:32:23"
},
"sitestatstable": {
"status": "done",
"files": {"elwiki-20170401-site_stats.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-site_stats.sql.gz",
"sha1": "169e86e117f2abb4f781b6fdc6138686c01a1cf4",
"md5": "7a704a4940e0bd70283b076f1c3cf1ac",
"size": 808}
},
"updated": "2017-04-02 09:15:50"
},
"articlesdump": {
"status": "done",
"files": {"elwiki-20170401-pages-articles.xml.bz2": {
"url": "/elwiki/20170401/elwiki-20170401-pages-articles.xml.bz2",
"sha1": "7116e38dddbdf20cf851269372d7dff82757547a",
"md5": "0592ed18210ef4add15c06af05913187",
"size": 259844537}
},
"updated": "2017-04-02 20:08:56"
},
"templatelinkstable": {
"status": "done",
"files": {"elwiki-20170401-templatelinks.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-templatelinks.sql.gz",
"sha1": "da84e2d054fb29323cd64d78d22b25aef397fc89",
"md5": "47bc4cca96d2cf05952797bee127c24d",
"size": 11590887}
},
"updated": "2017-04-02 09:16:24"
},
"geotagstable": {
"status": "done",
"files": {"elwiki-20170401-geo_tags.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-geo_tags.sql.gz",
"sha1": "f83179e884f752f004d61e4625d8f9a2d9fdcd1f",
"md5": "c32a4154bcd18ea446f1f2ae77ae7606",
"size": 353388}
},
"updated": "2017-04-02 09:17:30"
},
"categorylinkstable": {
"status": "done",
"files": {"elwiki-20170401-categorylinks.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-categorylinks.sql.gz",
"sha1": "b3871bba6467f3bf0d1170c31b9ae9a9d483e9bf",
"md5": "770f8a3228ae06de9f0704b66339a198",
"size": 20751244}
},
"updated": "2017-04-02 09:17:19"
},
"pagerestrictionstable": {
"status": "done",
"files": {"elwiki-20170401-page_restrictions.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-page_restrictions.sql.gz",
"sha1": "32b8dd16458c9297568dba39d35ef11778d74bcf",
"md5": "d8d03916d6473cfa86ae0c9d1da5a015",
"size": 2780}
},
"updated": "2017-04-02 09:15:26"
},
"namespaces": {
"status": "done",
"files": {"elwiki-20170401-siteinfo-namespaces.json": {
"url": "/elwiki/20170401/elwiki-20170401-siteinfo-namespaces.json",
"sha1": "fe5275cb6ece8d1bc7dd9b6d9510548281d4504e",
"md5": "6bd3b8139233a3c8c9e43abaab76dd02",
"size": 32742}
},
"updated": "2017-04-05 14:50:38"
},
"allpagetitlesdump": {
"status": "done",
"files": {"elwiki-20170401-all-titles.gz": {
"url": "/elwiki/20170401/elwiki-20170401-all-titles.gz",
"sha1": "92fa8deb8171aeb1f34901152e2f165aa0be0482",
"md5": "8301eb0515c4919033d5edf27c5c4b48",
"size": 3087383}
},
"updated": "2017-04-05 14:50:45"
},
"abstractsdump": {
"status": "done",
"files": {"elwiki-20170401-abstract.xml": {
"url": "/elwiki/20170401/elwiki-20170401-abstract.xml",
"sha1": "cd9ade3d9bc36b3f223b2d03d3c7f661de35d7de",
"md5": "76a6918bdd6402047d082a8c1c08f71a",
"size": 254782486}
},
"updated": "2017-04-05 16:03:03"
},
"pagetitlesdump": {
"status": "done",
"files": {"elwiki-20170401-all-titles-in-ns0.gz": {
"url": "/elwiki/20170401/elwiki-20170401-all-titles-in-ns0.gz",
"sha1": "db5309b3946b2025578feb1d7f88f3cf7ec52a0d",
"md5": "d4fb9941204ef455c7936c9db4b1187f",
"size": 1471608}
},
"updated": "2017-04-05 14:50:41"
},
"externallinkstable": {
"status": "done",
"files": {"elwiki-20170401-externallinks.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-externallinks.sql.gz",
"sha1": "59be511b7263e818986225fa2873ff7ef9884de2",
"md5": "1191c39facb047de5ebca5ee8ecebf16",
"size": 32456069}
},
"updated": "2017-04-02 09:16:42"
},
"xmlstubsdump": {
"status": "done",
"files": {
"elwiki-20170401-stub-meta-current.xml.gz": {
"url": "/elwiki/20170401/elwiki-20170401-stub-meta-current.xml.gz",
"sha1": "774e9695811b6d3b6a8ef1ddf694b66ab3431c9e",
"md5": "3354e9862048f6fce9c0e9fc404783df",
"size": 38735877
},
"elwiki-20170401-stub-meta-history.xml.gz": {
"url": "/elwiki/20170401/elwiki-20170401-stub-meta-history.xml.gz",
"sha1": "5ed749834b512e54759210d22f436e4acbb3316c",
"md5": "ae0644707770ffaa5b7671f2bbd18fd2",
"size": 435417311
},
"elwiki-20170401-stub-articles.xml.gz": {
"url": "/elwiki/20170401/elwiki-20170401-stub-articles.xml.gz",
"sha1": "43574bcefc29736e29681ba27ff39b37c6e03c44",
"md5": "120b087c738510458fa3189909b6f7a8",
"size": 27865308
}
},
"updated": "2017-04-02 12:18:04"
},
"redirecttable": {
"status": "done",
"files": {"elwiki-20170401-redirect.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-redirect.sql.gz",
"sha1": "ffddffc9164fa3e506b14e705c6da799dc1c92bd",
"md5": "a73ba8df1babd8cc11e6f62d59e233b6",
"size": 1017224}
},
"updated": "2017-04-02 09:15:52"
},
"wbcentityusagetable": {
"status": "done",
"files": {"elwiki-20170401-wbc_entity_usage.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-wbc_entity_usage.sql.gz",
"sha1": "872e87419370c2c0b7e67415cdd7e7665ac31237",
"md5": "64f41780be9d58fdff163ce1182197b8",
"size": 6824311}
},
"updated": "2017-04-02 09:15:23"
},
"metahistorybz2dump": {
"status": "done",
"files": {"elwiki-20170401-pages-meta-history.xml.bz2": {
"url": "/elwiki/20170401/elwiki-20170401-pages-meta-history.xml.bz2",
"sha1": "9110752456f98dd9f0178a479a08c8d4a23f2c92",
"md5": "5aae4de6df4801303fbb366e4796ae74",
"size": 5526431562}
},
"updated": "2017-04-06 09:15:45"
},
"sitestable": {
"status": "done",
"files": {"elwiki-20170401-sites.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-sites.sql.gz",
"sha1": "53a578167a3e19062126ab7312490115b48930bf",
"md5": "d76499ce3ba92c7c2723afa34aa99f0b",
"size": 19572}
},
"updated": "2017-04-02 09:15:45"
},
"categorytable": {
"status": "done",
"files": {"elwiki-20170401-category.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-category.sql.gz",
"sha1": "202fd445bf76b2f0b6c807cac473ebf349cf8c6a",
"md5": "d25e3d01dfa53d7559627bc8b41b2e76",
"size": 645888}
},
"updated": "2017-04-02 09:15:35"
},
"xmlflowdump": {
"status": "done",
"files": {"elwiki-20170401-flow.xml.bz2": {
"url": "/elwiki/20170401/elwiki-20170401-flow.xml.bz2",
"sha1": "d0718edf90e513f6fd1146d9f0982809eb62017b",
"md5": "c960c6d130e4232acfee064b14461214",
"size": 284603}
},
"updated": "2017-04-05 16:05:59"
},
"protectedtitlestable": {
"status": "done",
"files": {"elwiki-20170401-protected_titles.sql.gz": {
"url": "/elwiki/20170401/elwiki-20170401-protected_titles.sql.gz",
"sha1": "c3a8fbc6b01e1fec13eb2557cb49f99411584679",
"md5": "8df774774abaaa8825c4bb03bbba887f",
"size": 5197}
},
"updated": "2017-04-02 09:16:27"
}
}
}
Fields (description covers version 0.8)
- Version: version of this json format
- Jobs: one entry per dump job, containing the job name and various fields related to that job
- Status: status of the dump job ('done', 'in-progress', etc)
- Files: information on each file produced by the specific dump job
- URL: url of an output file relative to the HTML docroot
- SHA1: sha1 hash of the output file
- MD5: md5 hash of the output file
- Size: size in bytes of the output file
- Updated: when the information about the specific dump job was last updated
Fields may be missing if a dump job is in progress, has been aborted/failed or has not yet been run.