The schema array in the response lists each column’s name, Arrow type and description.
Provenance columns
These columns ship in the metadata of the helmet-camera dataset:
Column
What it holds
consent_version
Version of the consent agreement the contributor signed.
faces_blurred
Whether bystander faces were blurred.
plates_blurred
Whether number plates were blurred.
driver_id
Pseudonymous contributor ID. Split train and test on this column.
Querying the metadata
Parquet reads directly in DuckDB, Polars, pandas and PyArrow:
import duckdbduckdb.sql(""" SELECT city, round(sum(duration_s) / 3600) AS hours FROM 'datasets/sea-helmet-cam/metadata/*.parquet' WHERE weather = 'rain' GROUP BY city ORDER BY hours DESC""").show()
{"docsConfig":{"theme":"mint","$schema":"https://mintlify.com/docs.json","name":"datachad","colors":{"primary":"#1F4FB0","light":"#F5C518","dark":"#0D1422"},"navbar":{"links":[{"href":"https://api.datachad.xyz/docs","label":"Swagger UI"}],"primary":{"type":"button","label":"Get an API key","href":"https://datachad.xyz/signup"}},"footer":{"socials":{"website":"https://datachad.xyz"}},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"hasAudioNarration":false,"customDomains":[],"gitSource":{"type":"github","owner":"local","repo":"local","deployBranch":"main","contentDirectory":"","isPrivate":true,"mountPath":"","label":"local","isBase":true},"buildId":"local","clientVersion":"0.0.4173"},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[],"googleFontsInlined":false,"isLoggedInCli":false},"customCss":[],"page":{"pagePath":"guides/dataset-layout","slug":"/guides/dataset-layout","pageMetadata":{"title":"Dataset layout","description":"How a delivered dataset is organised.","href":"/guides/dataset-layout"},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsx(_components.p, {\n children: \"How a delivered dataset is organised.\"\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={code:`code`,p:`p`,pre:`pre`,span:`span`,tbody:`tbody`,td:`td`,th:`th`,thead:`thead`,tr:`tr`,..._provideComponents(),...props.components},{CodeBlock,Heading,Table}=_components;if(!CodeBlock)_missingMdxReference(`CodeBlock`,true);if(!Heading)_missingMdxReference(`Heading`,true);if(!Table)_missingMdxReference(`Table`,true);return _jsxs(_Fragment,{children:[_jsx(_components.p,{children:`Every dataset, licensed or commissioned, is delivered in the same layout: files in object storage and one Parquet metadata row per sample.`}),`\n`,_jsx(CodeBlock,{numberOfLines:`5`,language:`text`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`text`,children:_jsxs(_components.code,{language:`text`,numberOfLines:`5`,children:[_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`datasets/\u003cslug>/`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:` dataset.json title, description, licence, column descriptions`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:` metadata/*.parquet one row per sample`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:` clips/\u003ccity>/\u003cid>.mp4 the files themselves`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{})})]})})}),`\n`,_jsx(Heading,{level:`2`,id:`metadata`,children:`Metadata`}),`\n`,_jsx(_components.p,{children:`The metadata table is what you filter on before you load anything. Read a dataset’s columns from the API:`}),`\n`,_jsx(CodeBlock,{numberOfLines:`2`,language:`shellscript`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`shellscript`,children:_jsxs(_components.code,{language:`shellscript`,numberOfLines:`2`,children:[_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#953800`,\"--shiki-dark\":`#DCDCAA`},children:`curl`}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` https://api.datachad.xyz/v1/datasets/sea-helmet-cam`}),_jsx(_components.span,{style:{color:`#CF222E`,\"--shiki-dark\":`#D7BA7D`},children:` \\\\`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0550AE`,\"--shiki-dark\":`#569CD6`},children:` -H`}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"X-API-Key: `}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#9CDCFE`},children:`$DATACHAD_API_KEY`}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"`})]}),`\n`]})})}),`\n`,_jsxs(_components.p,{children:[`The `,_jsx(_components.code,{children:`schema`}),` array in the response lists each column’s name, Arrow type and description.`]}),`\n`,_jsx(Heading,{level:`2`,id:`provenance-columns`,children:`Provenance columns`}),`\n`,_jsx(_components.p,{children:`These columns ship in the metadata of the helmet-camera dataset:`}),`\n`,_jsxs(Table,{children:[_jsx(_components.thead,{children:_jsxs(_components.tr,{children:[_jsx(_components.th,{children:`Column`}),_jsx(_components.th,{children:`What it holds`})]})}),_jsxs(_components.tbody,{children:[_jsxs(_components.tr,{children:[_jsx(_components.td,{children:_jsx(_components.code,{children:`consent_version`})}),_jsx(_components.td,{children:`Version of the consent agreement the contributor signed.`})]}),_jsxs(_components.tr,{children:[_jsx(_components.td,{children:_jsx(_components.code,{children:`faces_blurred`})}),_jsx(_components.td,{children:`Whether bystander faces were blurred.`})]}),_jsxs(_components.tr,{children:[_jsx(_components.td,{children:_jsx(_components.code,{children:`plates_blurred`})}),_jsx(_components.td,{children:`Whether number plates were blurred.`})]}),_jsxs(_components.tr,{children:[_jsx(_components.td,{children:_jsx(_components.code,{children:`driver_id`})}),_jsx(_components.td,{children:`Pseudonymous contributor ID. Split train and test on this column.`})]})]})]}),`\n`,_jsx(Heading,{level:`2`,id:`querying-the-metadata`,children:`Querying the metadata`}),`\n`,_jsx(_components.p,{children:`Parquet reads directly in DuckDB, Polars, pandas and PyArrow:`}),`\n`,_jsx(CodeBlock,{numberOfLines:`9`,language:`python`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`python`,children:_jsxs(_components.code,{language:`python`,numberOfLines:`9`,children:[_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#CF222E`,\"--shiki-dark\":`#C586C0`},children:`import`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` duckdb`})]}),`\n`,_jsx(_components.span,{className:`line`}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`duckdb.sql(`}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"\"\"`})]}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` SELECT city, round(sum(duration_s) / 3600) AS hours`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` FROM 'datasets/sea-helmet-cam/metadata/*.parquet'`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` WHERE weather = 'rain'`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` GROUP BY city`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` ORDER BY hours DESC`})}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"\"\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`).show()`})]}),`\n`]})})})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Dataset layout","description":"How a delivered dataset is organised.","href":"/guides/dataset-layout"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[{"title":"Metadata","slug":"metadata","depth":2,"children":[]},{"title":"Provenance columns","slug":"provenance-columns","depth":2,"children":[]},{"title":"Querying the metadata","slug":"querying-the-metadata","depth":2,"children":[]}],"codeExamples":{}},"jsLinks":[],"subdomain":"docs.datachad.xyz","siteName":"datachad","title":"Dataset layout - datachad","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"originalFileLocation":"guides/dataset-layout.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"How a delivered dataset is organised.\">\u003cmeta data-mint-head name=\"application-name\" content=\"datachad\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/local/_generated/favicon/apple-touch-icon.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/local/_generated/favicon/favicon-32x32.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/local/_generated/favicon/favicon-16x16.png?v=3\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/local/_generated/favicon/favicon.ico?v=3\">\u003cmeta data-mint-head property=\"og:title\" content=\"Dataset layout - datachad\">\u003cmeta data-mint-head property=\"og:description\" content=\"How a delivered dataset is organised.\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"datachad\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://local./guides/dataset-layout\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"http://docs.datachad.xyz/_mintlify/og/guides/dataset-layout.png\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Dataset layout - datachad\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"How a delivered dataset is organised.\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"http://docs.datachad.xyz/_mintlify/og/guides/dataset-layout.png\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"https://mintlify.s3.us-west-1.amazonaws.com/local/_generated/favicon/browserconfig.xml?v=3\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"datachad\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#1F4FB0\">\u003clink data-mint-head rel=\"canonical\" href=\"https://local./guides/dataset-layout\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/guides/dataset-layout.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"datachad\",\"publisher\":{\"@type\":\"Organization\",\"name\":\"datachad\"},\"provider\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","agentPointerHref":"/guides/dataset-layout.md","siteName":"datachad","nav":{"tabs":[{"tab":"Guides","groups":[{"group":"Get started","pages":[{"title":"datachad API","href":""},{"title":"Quickstart","href":"/quickstart"}]},{"group":"Using the API","pages":[{"title":"Authentication","href":"/guides/authentication"},{"title":"API keys","href":"/guides/api-keys"},{"title":"Errors","description":"Status codes and the error body.","href":"/guides/errors"}]},{"group":"Working with datasets","pages":[{"title":"Dataset layout","href":"/guides/dataset-layout"},{"title":"Loading into PyTorch","description":"Filter the metadata, then decode only the clips you picked.","href":"/guides/pytorch"}]}]},{"tab":"API reference","pages":[{"title":"API reference","href":"/api-reference/auth/log-in"}]}]},"basePath":"","title":"Dataset layout - datachad","buildId":"805bd8087ee3","dynamics":[],"chunks":[]}