Data dumps/Status format

From Meta, a Wikimedia project coordination wiki
Jump to: navigation, search

We are now producing a json-formatted status file for the dump of each wiki on a particular date. It is updated as files are produced. Example format (indentation added for readability):


{"version": "0.8", 
 "jobs": { 
     "articlesmultistreamdump": { 
         "status": "done",
         "files": {
             "elwiki-20170401-pages-articles-multistream.xml.bz2": 
                   {"url": "/elwiki/20170401/elwiki-20170401-pages-articles-multistream.xml.bz2", "size": 271090061},
             "elwiki-20170401-pages-articles-multistream-index.txt.bz2":
                   {"url": "/elwiki/20170401/elwiki-20170401-pages-articles-multistream-index.txt.bz2", "size": 2717257}
            }, 
         "updated": "2017-04-03 14:31:20"},
     "langlinkstable": {
         "status": "done",
         "files": {"elwiki-20170401-langlinks.sql.gz": {
                         "url": "/elwiki/20170401/elwiki-20170401-langlinks.sql.gz",
                         "sha1": "6816faf2ddbe69fe92ecb7c71f061a1f18054d92",
                         "md5": "58c448d467801247ddcf4b2d0eddbee8",
                         "size": 44516915}
          },
          "updated": "2017-04-02 09:15:14"
     },
     "pagelinkstable": {
         "status": "done", 
         "files": {"elwiki-20170401-pagelinks.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-pagelinks.sql.gz",
                        "sha1": "272291561fb44d3b23427d455897d861c3d2cf1b",
                        "md5": "fc08a8a9e1914604f8370148e24d7c5d",
                        "size": 46811358}
         },
         "updated": "2017-04-02 09:18:53"}, 
     "imagetable": {
         "status": "done", 
         "files": {"elwiki-20170401-image.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-image.sql.gz",
                        "sha1": "0e1f299481842c9bb296c6adc8e072b7aa0d2b79",
                        "md5": "2ff70134e9a963aebbd90eb1a560dd54",
                        "size": 2216907}
          },
          "updated": "2017-04-02 09:15:19"
     }, 
     "iwlinkstable": {
         "status": "done",
         "files": {"elwiki-20170401-iwlinks.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-iwlinks.sql.gz",
                        "sha1": "f96864af070dfb7cfcf27223425c120e9a7a0943",
                        "md5": "6d3c6d016553058eee814621d3211b3c",
                        "size": 2204376}
          },
          "updated": "2017-04-02 09:16:31"
     }, 
     "usergroupstable": {
         "status": "done",
         "files": {"elwiki-20170401-user_groups.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-user_groups.sql.gz",
                        "sha1": "d99b7a23891602ad323b8e8bc93a08f675213dbc",
                        "md5": "277e05192dff59eaa4e6e4008cd810be",
                        "size": 1291}
         },
         "updated": "2017-04-02 09:15:47"
     }, 
     "xmlpagelogsdump": {
         "status": "done",
         "files": {"elwiki-20170401-pages-logging.xml.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-pages-logging.xml.gz",
                        "sha1": "257e62ddd008639edc69ab71449db28e081833df",
                        "md5": "c7307dd58b297d6220ba75342dbc8d53",
                        "size": 26669476}
         },
         "updated": "2017-04-05 16:05:30"
     }, 
     "changetagstable": {
         "status": "done", 
         "files": {"elwiki-20170401-change_tag.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-change_tag.sql.gz",
                        "sha1": "99e2321039f38db91eb29e99ca08d78b443ab94a",
                        "md5": "0e2a0d38516e4f7090705b0bf42c6582",
                        "size": 2264729}
         },
         "updated": "2017-04-02 09:17:21"
     }, 
     "imagelinkstable": {
         "status": "done",
         "files": {"elwiki-20170401-imagelinks.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-imagelinks.sql.gz",
                        "sha1": "8f8f2d264e97290b6dd523f0db2aa3c3ed91d458",
                        "md5": "f1720958a5a2bcff022da971d4d0d95b",
                        "size": 5331794}
         },
         "updated": "2017-04-02 09:15:43"
     }, 
     "pagetable": {
         "status": "done",
         "files": {"elwiki-20170401-page.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-page.sql.gz",
                        "sha1": "1dbea87e06ee22f989ef5c73365ed8a228c7d948", 
                        "md5": "d398b194eb5dc0cba604408b324f33d4", 
                        "size": 15013830}
         },
         "updated": "2017-04-02 09:15:33"
     }, 
     "xmlflowhistorydump": {
         "status": "done",
         "files": {"elwiki-20170401-flowhistory.xml.bz2": {
                        "url": "/elwiki/20170401/elwiki-20170401-flowhistory.xml.bz2", 
                        "sha1": "4670f375e65dbee01cd1ea98168bd3754d088ff4", 
                        "md5": "8ba8baf2b2b0b218eec6fc255dc2d19d", 
                        "size": 309637}
         },
         "updated": "2017-04-05 16:06:20"
     }, 
     "metacurrentdump": {
         "status": "done",
         "files": {"elwiki-20170401-pages-meta-current.xml.bz2": {
                        "url": "/elwiki/20170401/elwiki-20170401-pages-meta-current.xml.bz2", 
                        "sha1": "676756c1750e9643ba444d775493806c4e27ac4b", 
                        "md5": "e290fc403eb5f9818167ef2fbeeaac2e", 
                        "size": 326017055}
         },
         "updated": "2017-04-04 16:14:40"
     }, 
     "pagepropstable": {
         "status": "done",
         "files": {"elwiki-20170401-page_props.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-page_props.sql.gz", 
                        "sha1": "93bac8fc0968f2d200481e6ad8d3ef68dcaad403", 
                        "md5": "66e613d39c13aabd3e0803502348a039", 
                        "size": 5085458}
         },
         "updated": "2017-04-02 09:17:27"
     }, 
     "metahistory7zdump": {
         "status": "done",
         "files": {"elwiki-20170401-pages-meta-history.xml.7z": {
                        "url": "/elwiki/20170401/elwiki-20170401-pages-meta-history.xml.7z", 
                        "sha1": "ec7c2a4e61258b1d6e8d5924ac01261dc64af4ff", 
                        "md5": "43420adca10298f272ab5cd1977eb812", 
                        "size": 1102932726}
         },
         "updated": "2017-04-06 10:32:23"
     }, 
     "sitestatstable": {
         "status": "done",
         "files": {"elwiki-20170401-site_stats.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-site_stats.sql.gz", 
                        "sha1": "169e86e117f2abb4f781b6fdc6138686c01a1cf4", 
                        "md5": "7a704a4940e0bd70283b076f1c3cf1ac", 
                        "size": 808}
         },
         "updated": "2017-04-02 09:15:50"
     }, 
     "articlesdump": {
         "status": "done",
         "files": {"elwiki-20170401-pages-articles.xml.bz2": {
                        "url": "/elwiki/20170401/elwiki-20170401-pages-articles.xml.bz2", 
                        "sha1": "7116e38dddbdf20cf851269372d7dff82757547a", 
                        "md5": "0592ed18210ef4add15c06af05913187", 
                        "size": 259844537}
         },
         "updated": "2017-04-02 20:08:56"
     }, 
     "templatelinkstable": {
         "status": "done",
         "files": {"elwiki-20170401-templatelinks.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-templatelinks.sql.gz", 
                        "sha1": "da84e2d054fb29323cd64d78d22b25aef397fc89", 
                        "md5": "47bc4cca96d2cf05952797bee127c24d", 
                        "size": 11590887}
         },
         "updated": "2017-04-02 09:16:24"
     }, 
     "geotagstable": {
         "status": "done",
         "files": {"elwiki-20170401-geo_tags.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-geo_tags.sql.gz", 
                        "sha1": "f83179e884f752f004d61e4625d8f9a2d9fdcd1f", 
                        "md5": "c32a4154bcd18ea446f1f2ae77ae7606", 
                        "size": 353388}
         },
         "updated": "2017-04-02 09:17:30"
     }, 
     "categorylinkstable": {
         "status": "done", 
        "files": {"elwiki-20170401-categorylinks.sql.gz": {
                       "url": "/elwiki/20170401/elwiki-20170401-categorylinks.sql.gz", 
                       "sha1": "b3871bba6467f3bf0d1170c31b9ae9a9d483e9bf", 
                       "md5": "770f8a3228ae06de9f0704b66339a198", 
                       "size": 20751244}
         },
         "updated": "2017-04-02 09:17:19"
     }, 
     "pagerestrictionstable": {
         "status": "done", 
        "files": {"elwiki-20170401-page_restrictions.sql.gz": {
                       "url": "/elwiki/20170401/elwiki-20170401-page_restrictions.sql.gz", 
                       "sha1": "32b8dd16458c9297568dba39d35ef11778d74bcf", 
                       "md5": "d8d03916d6473cfa86ae0c9d1da5a015", 
                       "size": 2780}
        },
        "updated": "2017-04-02 09:15:26"
     }, 
     "namespaces": {
         "status": "done",
         "files": {"elwiki-20170401-siteinfo-namespaces.json": {
                        "url": "/elwiki/20170401/elwiki-20170401-siteinfo-namespaces.json", 
                        "sha1": "fe5275cb6ece8d1bc7dd9b6d9510548281d4504e", 
                        "md5": "6bd3b8139233a3c8c9e43abaab76dd02", 
                        "size": 32742}
         },
         "updated": "2017-04-05 14:50:38"
     }, 
     "allpagetitlesdump": {
         "status": "done",
         "files": {"elwiki-20170401-all-titles.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-all-titles.gz", 
                        "sha1": "92fa8deb8171aeb1f34901152e2f165aa0be0482", 
                        "md5": "8301eb0515c4919033d5edf27c5c4b48", 
                        "size": 3087383}
         },
         "updated": "2017-04-05 14:50:45"
     }, 
     "abstractsdump": {
         "status": "done",
         "files": {"elwiki-20170401-abstract.xml": {
                        "url": "/elwiki/20170401/elwiki-20170401-abstract.xml", 
                        "sha1": "cd9ade3d9bc36b3f223b2d03d3c7f661de35d7de", 
                        "md5": "76a6918bdd6402047d082a8c1c08f71a", 
                        "size": 254782486}
         },
         "updated": "2017-04-05 16:03:03"
     }, 
     "pagetitlesdump": {
         "status": "done",
         "files": {"elwiki-20170401-all-titles-in-ns0.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-all-titles-in-ns0.gz", 
                        "sha1": "db5309b3946b2025578feb1d7f88f3cf7ec52a0d", 
                        "md5": "d4fb9941204ef455c7936c9db4b1187f", 
                        "size": 1471608}
         },
         "updated": "2017-04-05 14:50:41"
     }, 
     "externallinkstable": {
         "status": "done",
         "files": {"elwiki-20170401-externallinks.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-externallinks.sql.gz", 
                        "sha1": "59be511b7263e818986225fa2873ff7ef9884de2", 
                        "md5": "1191c39facb047de5ebca5ee8ecebf16", 
                        "size": 32456069}
         },
         "updated": "2017-04-02 09:16:42"
     }, 
     "xmlstubsdump": {
         "status": "done",
         "files": {
             "elwiki-20170401-stub-meta-current.xml.gz": {
                 "url": "/elwiki/20170401/elwiki-20170401-stub-meta-current.xml.gz",
                 "sha1": "774e9695811b6d3b6a8ef1ddf694b66ab3431c9e",
                 "md5": "3354e9862048f6fce9c0e9fc404783df",
                 "size": 38735877
             },
             "elwiki-20170401-stub-meta-history.xml.gz": {
                 "url": "/elwiki/20170401/elwiki-20170401-stub-meta-history.xml.gz",
                 "sha1": "5ed749834b512e54759210d22f436e4acbb3316c",
                 "md5": "ae0644707770ffaa5b7671f2bbd18fd2",
                 "size": 435417311
             },
             "elwiki-20170401-stub-articles.xml.gz": {
                 "url": "/elwiki/20170401/elwiki-20170401-stub-articles.xml.gz",
                 "sha1": "43574bcefc29736e29681ba27ff39b37c6e03c44",
                 "md5": "120b087c738510458fa3189909b6f7a8",
                 "size": 27865308
             }
         },
         "updated": "2017-04-02 12:18:04"
     }, 
     "redirecttable": {
         "status": "done",
         "files": {"elwiki-20170401-redirect.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-redirect.sql.gz",
                        "sha1": "ffddffc9164fa3e506b14e705c6da799dc1c92bd",
                        "md5": "a73ba8df1babd8cc11e6f62d59e233b6",
                        "size": 1017224}
          },
          "updated": "2017-04-02 09:15:52"
     }, 
     "wbcentityusagetable": {
         "status": "done",
         "files": {"elwiki-20170401-wbc_entity_usage.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-wbc_entity_usage.sql.gz",
                        "sha1": "872e87419370c2c0b7e67415cdd7e7665ac31237",
                        "md5": "64f41780be9d58fdff163ce1182197b8",
                        "size": 6824311}
          },
          "updated": "2017-04-02 09:15:23"
     }, 
     "metahistorybz2dump": {
         "status": "done",
         "files": {"elwiki-20170401-pages-meta-history.xml.bz2": {
                        "url": "/elwiki/20170401/elwiki-20170401-pages-meta-history.xml.bz2",
                        "sha1": "9110752456f98dd9f0178a479a08c8d4a23f2c92",
                        "md5": "5aae4de6df4801303fbb366e4796ae74",
                        "size": 5526431562}
         },
         "updated": "2017-04-06 09:15:45"
     }, 
     "sitestable": {
         "status": "done",
         "files": {"elwiki-20170401-sites.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-sites.sql.gz",
                        "sha1": "53a578167a3e19062126ab7312490115b48930bf",
                        "md5": "d76499ce3ba92c7c2723afa34aa99f0b",
                        "size": 19572}
         },
         "updated": "2017-04-02 09:15:45"
     },
     "categorytable": {
         "status": "done",
         "files": {"elwiki-20170401-category.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-category.sql.gz",
                        "sha1": "202fd445bf76b2f0b6c807cac473ebf349cf8c6a",
                        "md5": "d25e3d01dfa53d7559627bc8b41b2e76",
                        "size": 645888}
          },
          "updated": "2017-04-02 09:15:35"
     }, 
     "xmlflowdump": {
         "status": "done",
         "files": {"elwiki-20170401-flow.xml.bz2": {
                        "url": "/elwiki/20170401/elwiki-20170401-flow.xml.bz2",
                        "sha1": "d0718edf90e513f6fd1146d9f0982809eb62017b",
                        "md5": "c960c6d130e4232acfee064b14461214",
                        "size": 284603}
         },
         "updated": "2017-04-05 16:05:59"
     }, 
     "protectedtitlestable": {
         "status": "done",
         "files": {"elwiki-20170401-protected_titles.sql.gz": {
                        "url": "/elwiki/20170401/elwiki-20170401-protected_titles.sql.gz",
                        "sha1": "c3a8fbc6b01e1fec13eb2557cb49f99411584679",
                        "md5": "8df774774abaaa8825c4bb03bbba887f",
                        "size": 5197}
         },
         "updated": "2017-04-02 09:16:27"
     }
  }
}

Fields (description covers version 0.8)

  • Version: version of this json format
  • Jobs: one entry per dump job, containing the job name and various fields related to that job
  • Status: status of the dump job ('done', 'in-progress', etc)
  • Files: information on each file produced by the specific dump job
  • URL: url of an output file relative to the HTML docroot
  • SHA1: sha1 hash of the output file
  • MD5: md5 hash of the output file
  • Size: size in bytes of the output file
  • Updated: when the information about the specific dump job was last updated

Fields may be missing if a dump job is in progress, has been aborted/failed or has not yet been run.