1:HL["/_next/static/media/6905431624c34d00-s.p.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
2:HL["/_next/static/css/9e925a33b1acdac1.css","style",{"crossOrigin":""}]
0:["rmcKjFZ3e9kKdH1iJwCIQ",[[["",{"children":["blog",{"children":[["slug","2023/05/30/how-to-ingest-streaming-data-from-kafka-to-apache-pinot","c"],{"children":["__PAGE__?{\"slug\":[\"2023\",\"05\",\"30\",\"how-to-ingest-streaming-data-from-kafka-to-apache-pinot\"]}",{}]}]}]},"$undefined","$undefined",true],"$L3",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/9e925a33b1acdac1.css","precedence":"next","crossOrigin":""}]],"$L4"]]]]
5:HL["/_next/static/css/c130d1629644f070.css","style",{"crossOrigin":""}]
6:I[7821,["326","static/chunks/326-3a90a6443b9c824c.js","980","static/chunks/980-6e243f9cd384c7d2.js","702","static/chunks/702-a2bf9fe707814b79.js","185","static/chunks/app/layout-776a485845c720ef.js"],"ThemeProviders"]
7:I[3994,["326","static/chunks/326-3a90a6443b9c824c.js","980","static/chunks/980-6e243f9cd384c7d2.js","702","static/chunks/702-a2bf9fe707814b79.js","185","static/chunks/app/layout-776a485845c720ef.js"],""]
8:I[9640,["326","static/chunks/326-3a90a6443b9c824c.js","980","static/chunks/980-6e243f9cd384c7d2.js","702","static/chunks/702-a2bf9fe707814b79.js","185","static/chunks/app/layout-776a485845c720ef.js"],"AlgoliaSearchProvider"]
9:I[7975,["326","static/chunks/326-3a90a6443b9c824c.js","980","static/chunks/980-6e243f9cd384c7d2.js","702","static/chunks/702-a2bf9fe707814b79.js","185","static/chunks/app/layout-776a485845c720ef.js"],""]
a:I[6954,[],""]
b:I[7264,[],""]
c:I[8326,["326","static/chunks/326-3a90a6443b9c824c.js","413","static/chunks/413-f9f40b83f7bb3f22.js","980","static/chunks/980-6e243f9cd384c7d2.js","797","static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js"],""]
f:T9fe,M42.99 18.448c1.032-.553 2.21-.831 3.535-.831 1.542 0 2.938.38 4.187 1.14 1.248.76 2.236 1.841 2.965 3.241.728 1.402 1.091 3.025 1.091 4.872s-.363 3.482-1.091 4.903c-.729 1.424-1.717 2.525-2.965 3.307-1.25.782-2.645 1.173-4.187 1.173-1.325 0-2.493-.271-3.503-.815-1.01-.543-1.83-1.226-2.46-2.053v14.612H36V17.912h4.562v2.606c.586-.825 1.395-1.515 2.426-2.068l.002-.002m6.452 5.605c-.445-.793-1.032-1.395-1.76-1.808a4.72 4.72 0 0 0-2.362-.618c-.847 0-1.602.211-2.33.635-.728.423-1.315 1.038-1.76 1.841-.445.804-.668 1.749-.668 2.835 0 1.087.221 2.032.668 2.835.445.804 1.032 1.417 1.76 1.842a4.557 4.557 0 0 0 2.33.635 4.57 4.57 0 0 0 2.362-.652c.728-.435 1.313-1.053 1.76-1.856.445-.804.668-1.76.668-2.867s-.223-2.025-.668-2.818v-.004M62.947 17.912v18.051h-4.562V17.912h4.562m.551-6.079a2.833 2.833 0 1 1-5.666 0 2.833 2.833 0 0 1 5.666 0M82.954 19.687c1.325 1.358 1.988 3.253 1.988 5.685v10.59H80.38v-9.97c0-1.434-.358-2.537-1.075-3.307-.717-.772-1.695-1.157-2.933-1.157-1.239 0-2.254.387-2.982 1.157-.728.772-1.091 1.873-1.091 3.307v9.97h-4.562V17.91h4.562v2.248a6.322 6.322 0 0 1 2.33-1.841c.944-.445 1.981-.669 3.111-.669 2.15 0 3.889.68 5.214 2.037v.002M92.892 35.098c-1.39-.77-2.482-1.861-3.275-3.275-.794-1.411-1.19-3.041-1.19-4.888s.406-3.475 1.221-4.888a8.502 8.502 0 0 1 3.34-3.275c1.412-.772 2.987-1.157 4.725-1.157 1.739 0 3.312.387 4.725 1.157a8.5 8.5 0 0 1 3.34 3.275c.815 1.411 1.222 3.041 1.222 4.888s-.418 3.475-1.255 4.888a8.708 8.708 0 0 1-3.388 3.275c-1.424.772-3.014 1.157-4.774 1.157-1.76 0-3.301-.385-4.691-1.157m7.021-3.421c.729-.402 1.309-1.005 1.744-1.809.435-.803.651-1.781.651-2.933 0-1.715-.451-3.035-1.351-3.958-.902-.924-2.004-1.385-3.307-1.385s-2.395.461-3.275 1.385c-.88.923-1.32 2.243-1.32 3.958 0 1.715.428 3.035 1.287 3.958.858.924 1.938 1.385 3.241 1.385.825 0 1.602-.2 2.33-.603v.002M115.96 21.658v8.734c0 .608.147 1.048.44 1.32.293.271.787.406 1.482.406H120v3.845h-2.867c-3.845 0-5.766-1.868-5.766-5.605v-8.7h-2.15v-3.746h2.15V13l4.595-1v5.912h4.04v3.746h-4.042M20.03 46.757l-5.538-1.385A1.97 1.97 0 0 1 13 43.46v-5.462c0-.841.349-1.601.907-2.146a12.212 12.212 0 0 0 6.975-3.644c2.602-2.731 3.627-6.578 2.882-10.251L21 9h-4V4a1 1 0 0 0-2 0v7a1 1 0 0 1-2 0v-1a1 1 0 0 0-2 0v6.758a4.489 4.489 0 0 1 2.694-.755c2.278.095 4.156 1.934 4.297 4.21a4.501 4.501 0 0 1-6.992 4.029V29a1 1 0 0 1-2 0V7a1 1 0 0 0-2 0v2h-4L.237 21.957c-.745 3.675.279 7.52 2.882 10.251a12.202 12.202 0 0 0 6.975 3.644c.558.545.907 1.305.907 2.146V43.4c0 .938-.639 1.757-1.55 1.985l-5.48 1.37c-.57.143-.97.655-.97 1.243h18c0-.588-.4-1.1-.97-1.243v.0023:[null,["$","html",null,{"lang":"en-us","className":"__variable_1fc36d scroll-smooth","suppressHydrationWarning":true,"children":[["$","head",null,{"children":[["$","meta",null,{"httpEquiv":"Content-Security-Policy","content":"default-src 'self';script-src 'self' 'unsafe-eval' 'unsafe-inline' giscus.app analytics.umami.is www.youtube.com www.googletagmanager.com www.google-analytics.com;style-src 'self' 'unsafe-inline';img-src * blob: data:;media-src *.s3.amazonaws.com;connect-src *;font-src 'self';frame-src www.youtube.com youtube.com giscus.app youtu.be https://www.youtube.com https://youtube.com;"}],["$","link",null,{"rel":"apple-touch-icon","sizes":"76x76","href":"/static/favicons/apple-touch-icon.png"}],["$","link",null,{"rel":"icon","type":"image/png","sizes":"32x32","href":"/static/favicons/favicon-32x32.png"}],["$","link",null,{"rel":"icon","type":"image/png","sizes":"16x16","href":"/static/favicons/favicon-16x16.png"}],["$","link",null,{"rel":"manifest","href":"/static/favicons/site.webmanifest"}],["$","link",null,{"rel":"mask-icon","href":"/static/favicons/safari-pinned-tab.svg","color":"#5bbad5"}],["$","meta",null,{"name":"msapplication-TileColor","content":"#000000"}],["$","meta",null,{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}],["$","meta",null,{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#000"}],["$","link",null,{"rel":"alternate","type":"application/rss+xml","href":"/feed.xml"}]]}],["$","body",null,{"className":"bg-white text-black antialiased dark:bg-gray-950 dark:text-white","children":["$","$L6",null,{"children":[["$undefined","$undefined","$undefined","$undefined",[["$","$L7",null,{"strategy":"afterInteractive","src":"https://www.googletagmanager.com/gtag/js?id=G-ZXG79NJEBY"}],["$","$L7",null,{"strategy":"afterInteractive","id":"ga-script","children":"\n            window.dataLayer = window.dataLayer || [];\n            function gtag(){dataLayer.push(arguments);}\n            gtag('js', new Date());\n            gtag('config', 'G-ZXG79NJEBY');\n        "}]]],["$","div",null,{"className":"mx-auto flex max-w-screen-customDesktop flex-col justify-between font-sans","children":["$","$L8",null,{"algoliaConfig":{"appId":"CKRA00L2X9","apiKey":"6531f8f7783a88d76629190843f1801e","indexName":"prod_apache_pinot_docs"},"children":[["$","$L9",null,{}],["$","main",null,{"children":["$","$La",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$Lb",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":["$","div",null,{"className":"flex flex-col items-start justify-start md:mt-24 md:flex-row md:items-center md:justify-center md:space-x-6","children":[["$","div",null,{"className":"space-x-2 pb-8 pt-6 md:space-y-5","children":["$","h1",null,{"className":"text-6xl font-extrabold leading-9 tracking-tight text-gray-900 dark:text-gray-100 md:border-r-2 md:px-6 md:text-8xl md:leading-14","children":"404"}]}],["$","div",null,{"className":"max-w-md","children":[["$","p",null,{"className":"mb-4 text-xl font-bold leading-normal md:text-2xl","children":"Sorry we couldn't find this page."}],["$","p",null,{"className":"mb-8","children":"But dont worry, you can find plenty of other things on our homepage."}],["$","$Lc",null,{"href":"/","className":"focus:shadow-outline-blue inline rounded-lg border border-transparent bg-blue-600 px-4 py-2 text-sm font-medium leading-5 text-white shadow transition-colors duration-150 hover:bg-blue-700 focus:outline-none dark:hover:bg-blue-500","children":"Back to homepage"}]]}]]}],"notFoundStyles":[],"initialChildNode":["$","$La",null,{"parallelRouterKey":"children","segmentPath":["children","blog","children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$Lb",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","initialChildNode":["$","$La",null,{"parallelRouterKey":"children","segmentPath":["children","blog","children",["slug","2023/05/30/how-to-ingest-streaming-data-from-kafka-to-apache-pinot","c"],"children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$Lb",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","initialChildNode":["$Ld","$Le",null],"childPropSegment":"__PAGE__?{\"slug\":[\"2023\",\"05\",\"30\",\"how-to-ingest-streaming-data-from-kafka-to-apache-pinot\"]}","styles":[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/c130d1629644f070.css","precedence":"next","crossOrigin":""}]]}],"childPropSegment":["slug","2023/05/30/how-to-ingest-streaming-data-from-kafka-to-apache-pinot","c"],"styles":null}],"childPropSegment":"blog","styles":null}]}],["$","footer",null,{"className":"border-t bg-sky-100 px-5 py-10 md:px-[6.75rem] md:pb-10 md:pt-16","children":[["$","div",null,{"className":"mx-auto flex max-w-7xl flex-wrap justify-between","children":[["$","div",null,{"className":"flex-shrink-0","children":["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":120,"height":48,"fill":"none","children":[["$","g",null,{"fill":"#C7154A","clipPath":"url(#logo_svg__a)","children":[["$","path",null,{"d":"$f"}],["$","path",null,{"d":"M13.5 23a2.5 2.5 0 1 0 0-5 2.5 2.5 0 0 0 0 5M8 5a1 1 0 1 0 0-2 1 1 0 0 0 0 2M12 8a1 1 0 1 0 0-2 1 1 0 0 0 0 2M16 2a1 1 0 1 0 0-2 1 1 0 0 0 0 2"}]]}],["$","defs",null,{"children":["$","clipPath",null,{"id":"logo_svg__a","children":["$","path",null,{"fill":"#fff","d":"M0 0h120v48H0z"}]}]}]]}]}],["$","div",null,{"className":"flex flex-wrap gap-x-16 gap-y-5 py-8 md:pl-24 md:pr-[21.625rem]","children":[" ",[["$","div","Resources",{"children":[["$","h5",null,{"className":"mb-4 text-lg font-semibold","children":"Resources"}],["$","div",null,{"className":"flex justify-between gap-x-10","children":[["$","div",null,{"className":"flex flex-col","children":[["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://docs.pinot.apache.org/","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Docs"}],["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://docs.pinot.apache.org/getting-started","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Getting Started"}],["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://docs.pinot.apache.org/integrations/thirdeye","className":"block py-1 text-gray-600 hover:text-gray-900","children":"ThirdEye"}]]}],["$","div",null,{"className":"flex flex-col","children":[["$","$Lc",null,{"href":"/powered-by","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Company Stories"}],["$","$Lc",null,{"href":"/download","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Download"}],["$","$Lc",null,{"href":"/blog","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Blog"}]]}]]}]]}],["$","div","Apache",{"children":[["$","h5",null,{"className":"mb-4 text-lg font-semibold","children":"Apache"}],["$","div",null,{"className":"flex justify-between gap-x-10","children":[["$","div",null,{"className":"flex flex-col","children":[["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://www.apache.org","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Foundation"}],["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://www.apache.org/licenses","className":"block py-1 text-gray-600 hover:text-gray-900","children":"License"}],["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://www.apache.org/security","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Security"}]]}],["$","div",null,{"className":"flex flex-col","children":[["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://www.apache.org/foundation/sponsorship.html","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Sponsorship"}],["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://www.apache.org/events/current-event","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Events"}],["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://www.apache.org/foundation/thanks.html","className":"block py-1 text-gray-600 hover:text-gray-900","children":"Thanks"}]]}]]}]]}]]]}],["$","div",null,{"className":"mt-4 flex justify-center md:mt-0","children":[["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw","className":"mr-4","children":["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-slack fill-gray-900","children":[["$","rect","diqz80",{"width":"3","height":"8","x":"13","y":"2","rx":"1.5"}],["$","path","183iwg",{"d":"M19 8.5V10h1.5A1.5 1.5 0 1 0 19 8.5"}],["$","rect","hqg7r1",{"width":"3","height":"8","x":"8","y":"14","rx":"1.5"}],["$","path","76g71w",{"d":"M5 15.5V14H3.5A1.5 1.5 0 1 0 5 15.5"}],["$","rect","1kmz0a",{"width":"8","height":"3","x":"14","y":"13","rx":"1.5"}],["$","path","jc4sz0",{"d":"M15.5 19H14v1.5a1.5 1.5 0 1 0 1.5-1.5"}],["$","rect","1omvl4",{"width":"8","height":"3","x":"2","y":"8","rx":"1.5"}],["$","path","16f3cl",{"d":"M8.5 5H10V3.5A1.5 1.5 0 1 0 8.5 5"}],"$undefined"]}]}],["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://github.com/apache/pinot","children":["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"fill":"currentColor","size":24,"children":[["$","g",null,{"clipPath":"url(#github_svg__a)","children":["$","path",null,{"fillRule":"evenodd","d":"M12.01 0C5.369 0 0 5.5 0 12.304c0 5.44 3.44 10.043 8.212 11.673.597.122.815-.265.815-.59 0-.286-.02-1.264-.02-2.283-3.34.734-4.036-1.466-4.036-1.466-.537-1.426-1.332-1.793-1.332-1.793-1.094-.754.08-.754.08-.754 1.212.082 1.849 1.263 1.849 1.263 1.073 1.874 2.803 1.345 3.5 1.019.098-.795.417-1.345.755-1.65-2.665-.285-5.468-1.345-5.468-6.07 0-1.345.477-2.445 1.232-3.3-.119-.306-.537-1.57.12-3.26 0 0 1.014-.326 3.3 1.263.98-.27 1.989-.407 3.003-.408 1.014 0 2.048.143 3.002.408 2.287-1.59 3.301-1.263 3.301-1.263.657 1.69.239 2.954.12 3.26.775.855 1.232 1.955 1.232 3.3 0 4.725-2.803 5.764-5.488 6.07.438.387.815 1.12.815 2.281 0 1.65-.02 2.975-.02 3.382 0 .326.22.713.816.59C20.56 22.347 24 17.744 24 12.305 24.02 5.5 18.63 0 12.01 0","clipRule":"evenodd"}]}],["$","defs",null,{"children":["$","clipPath",null,{"id":"github_svg__a","children":["$","path",null,{"fill":"#fff","d":"M0 0h24v24H0z"}]}]}]]}]}]]}]]}],["$","div",null,{"className":"mt-8 border-t border-neutral-300 pt-4 text-left text-sm text-gray-600","children":["Copyright © ",2024," The Apache Software Foundation. Apache Pinot, Pinot, Apache, the Apache feather logo, and the Apache Pinot project logo are registered trademarks of The Apache Software Foundation. This page has references to third party software - Presto, PrestoDB, ThirdEye, Trino, TrinoDB, that are not part of the Apache Software Foundation and are not covered under the Apache License."]}]]}]]}]}]]}]}]]}],null]
4:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"How to Ingest Streaming Data from Kafka to Apache Pinot™ | Apache Pinot™"}],["$","meta","3",{"name":"description","content":"The blog post explains how to use Apache Kafka topics in Apache Pinot to ingest streaming data, with step-by-step instructions provided for installation and setup. It focuses on ingesting Wikipedia events into Kafka and connecting it to Pinot to create a real-time table. The post highlights Pinot's capabilities in ingesting and transforming JSON data into OLAP tables and encourages reader engagement through the community Slack."}],["$","meta","4",{"name":"robots","content":"index, follow"}],["$","meta","5",{"name":"googlebot","content":"index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1"}],["$","link","6",{"rel":"canonical","href":"https://pinot.apache.org/blog/2023/05/30/how-to-ingest-streaming-data-from-kafka-to-apache-pinot"}],["$","link","7",{"rel":"alternate","type":"application/rss+xml","href":"https://pinot.apache.org/feed.xml"}],["$","meta","8",{"property":"og:title","content":"How to Ingest Streaming Data from Kafka to Apache Pinot™"}],["$","meta","9",{"property":"og:description","content":"The blog post explains how to use Apache Kafka topics in Apache Pinot to ingest streaming data, with step-by-step instructions provided for installation and setup. It focuses on ingesting Wikipedia events into Kafka and connecting it to Pinot to create a real-time table. The post highlights Pinot's capabilities in ingesting and transforming JSON data into OLAP tables and encourages reader engagement through the community Slack."}],["$","meta","10",{"property":"og:url","content":"https://pinot.apache.org/blog/2023/05/30/how-to-ingest-streaming-data-from-kafka-to-apache-pinot"}],["$","meta","11",{"property":"og:site_name","content":"Apache Pinot™"}],["$","meta","12",{"property":"og:locale","content":"en_US"}],["$","meta","13",{"property":"og:image","content":"https://pinot.apache.org/static/images/twitter-card.png"}],["$","meta","14",{"property":"og:type","content":"article"}],["$","meta","15",{"property":"article:published_time","content":"2022-05-30T00:00:00.000Z"}],["$","meta","16",{"property":"article:modified_time","content":"2022-05-30T00:00:00.000Z"}],["$","meta","17",{"property":"article:author","content":"Barkha Herman"}],["$","meta","18",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","19",{"name":"twitter:title","content":"How to Ingest Streaming Data from Kafka to Apache Pinot™"}],["$","meta","20",{"name":"twitter:description","content":"The blog post explains how to use Apache Kafka topics in Apache Pinot to ingest streaming data, with step-by-step instructions provided for installation and setup. It focuses on ingesting Wikipedia events into Kafka and connecting it to Pinot to create a real-time table. The post highlights Pinot's capabilities in ingesting and transforming JSON data into OLAP tables and encourages reader engagement through the community Slack."}],["$","meta","21",{"name":"twitter:image","content":"https://pinot.apache.org/static/images/twitter-card.png"}],["$","meta","22",{"name":"next-size-adjust"}]]
10:I[1514,["326","static/chunks/326-3a90a6443b9c824c.js","413","static/chunks/413-f9f40b83f7bb3f22.js","980","static/chunks/980-6e243f9cd384c7d2.js","797","static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js"],""]
11:I[2529,["326","static/chunks/326-3a90a6443b9c824c.js","413","static/chunks/413-f9f40b83f7bb3f22.js","980","static/chunks/980-6e243f9cd384c7d2.js","797","static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js"],""]
12:I[5185,["326","static/chunks/326-3a90a6443b9c824c.js","413","static/chunks/413-f9f40b83f7bb3f22.js","980","static/chunks/980-6e243f9cd384c7d2.js","797","static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js"],""]
e:[["$","script",null,{"type":"application/ld+json","dangerouslySetInnerHTML":{"__html":"{\"@context\":\"https://schema.org\",\"@type\":\"BlogPosting\",\"headline\":\"How to Ingest Streaming Data from Kafka to Apache Pinot™\",\"datePublished\":\"2022-05-30T00:00:00.000Z\",\"dateModified\":\"2022-05-30T00:00:00.000Z\",\"description\":\"The blog post explains how to use Apache Kafka topics in Apache Pinot to ingest streaming data, with step-by-step instructions provided for installation and setup. It focuses on ingesting Wikipedia events into Kafka and connecting it to Pinot to create a real-time table. The post highlights Pinot's capabilities in ingesting and transforming JSON data into OLAP tables and encourages reader engagement through the community Slack.\",\"image\":\"/static/images/twitter-card.png\",\"url\":\"https://pinot.apache.org/blog/2023-05-30-how-to-ingest-streaming-data-from-kafka-to-apache-pinot\",\"author\":[{\"@type\":\"Person\",\"name\":\"Barkha Herman\"}]}"}}],["$","section",null,{"className":" px-5 pt-10 md:px-[13.313rem] md:py-16","children":[["$","$L10",null,{}],["$","article",null,{"className":"","children":["$","div",null,{"className":"mx-auto lg:flex","children":[["$","div",null,{"className":"lg:pr-12","children":[["$","header",null,{"className":"pt-6 md:pr-10","children":[["$","h1",null,{"className":"text-4xl font-semibold","children":"How to Ingest Streaming Data from Kafka to Apache Pinot™"}],["$","p",null,{"className":"pt-2 text-lg","children":["By: ","Barkha Herman"]}],["$","p",null,{"className":"py-2 text-sm","children":["May 30th, 2022"," • ","9 min read"]}]]}],["$","div",null,{"className":"flex flex-col lg:flex-row","children":["$","main",null,{"className":"","children":["$","div",null,{"className":"prose max-w-[45rem] pb-8 pt-10 dark:prose-invert","children":[["$","p",null,{"children":["We previously walked through getting started with ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://startree.ai/resources/what-is-apache-pinot","children":"Apache Pinot™"}]," using batch data, and now we will learn how to ingest streaming data using Apache Kafka® topics."]}],["$","p",null,{"children":"As the story goes, Apache Pinot was created at LinkedIn to provide a platform that could ingest a high number of incoming events (kafka) and provide “fresh” (sub second) analytics to a large number (20+ million) of users, fast (sub second latency). So, really, consuming events is part of the reason why Pinot was created."}],["$","h3",null,{"id":"the-obligatory-what-is-apache-pinot-and-startree-section","children":[["$","a",null,{"href":"#the-obligatory-what-is-apache-pinot-and-startree-section","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"The obligatory “What is Apache Pinot and StarTree?” section"]}],["$","p",null,{"children":[["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://docs.pinot.apache.org/","children":"Pinot"}]," is a real-time, distributed, open source, and free-to-use OLAP datastore, purpose-built to provide ultra low-latency analytics at extremely high throughput. It is open source and free to use."]}],["$","p",null,{"children":["How does StarTree come in? StarTree offers a ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://startree.ai/saas-signup","children":"fully managed version of the Apache Pinot real-time analytics system"}]," , plus other tools around it that you can try for free. The system includes  ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://dev.startree.ai/docs/startree-enterprise-edition/startree-dataset-manager/","children":"StarTree Dataset Manager"}]," and ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://dev.startree.ai/docs/procedures/get-started-with-thirdeye/","children":"StarTree ThirdEye"}],", a UI based data ingestion tool, and a real-time anomaly detection and root cause analysis tool, respectively."]}],["$","h2",null,{"id":"how-to-install-kafka-alongside-pinot","children":[["$","a",null,{"href":"#how-to-install-kafka-alongside-pinot","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"How to install Kafka alongside Pinot "]}],["$","h3",null,{"id":"prerequisite","children":[["$","a",null,{"href":"#prerequisite","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Prerequisite"]}],["$","p",null,{"children":["Complete the steps outlined in the ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://startree.ai/blog/apache-pinot-tutorial-for-getting-started-a-step-by-step-guide","children":"introduction to Apache Pinot"}],"."]}],["$","h3",null,{"id":"step-1-install-kafka-on-your-pinot-docker-image","children":[["$","a",null,{"href":"#step-1-install-kafka-on-your-pinot-docker-image","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Step 1: Install Kafka on your Pinot Docker image"]}],["$","p",null,{"children":"Make sure you have completed the first article in the series."}],["$","p",null,{"children":"We will be installing Apache Kafka onto our already existing Pinot docker image. To start the Docker image, run the following command:"}],["$","p",null,{"children":"docker run -it --entrypoint /bin/bash -p 9000:9000 apachepinot/pinot:0.12.0"}],["$","p",null,{"children":["$","img",null,{"alt":"PowerShell 7.3.4 docker run Apache Pinot","src":"https://www.datocms-assets.com/75153/1685462020-image1.png","title":"PowerShell 7.3.4 docker run Apache Pinot"}]}],["$","p",null,{"children":"We want to override the ENTRYPOINT and run Bash script within the Docker image. If you already have a container running, you can skip this step. I tend to tear down containers after use, so in my case, I created a brand new container."}],["$","p",null,{"children":"Now, start each of the components one at a time like we did in the previous session:"}],["$","p",null,{"children":"bin/pinot-admin.sh StartZookeeper &"}],["$","p",null,{"children":"bin/pinot-admin.sh StartController &"}],["$","p",null,{"children":"bin/pinot-admin.sh StartBroker &"}],["$","p",null,{"children":"bin/pinot-admin.sh StartServer &"}],["$","p",null,{"children":"Run each of the commands one at a time. The & allows you to continue using the same Bash shell session. If you like, you can create different shells for each service:"}],["$","ol",null,{"children":[["$","li",null,{"children":"Get the container ID by running docker ps"}],["$","li",null,{"children":["Run ",["$","code",null,{"children":"docker exec -it DOCKER_CONTAINER_ID bash"}]," where DOCKER_CONTAINER_ID is the ID received from step 1."]}],["$","li",null,{"children":"Run the pinot-admin.sh command to start the desired service"}]]}],["$","p",null,{"children":"It should look like this:"}],["$","p",null,{"children":["$","img",null,{"alt":"Docker with container ID, Image, Command, and Created","src":"https://www.datocms-assets.com/75153/1685462274-image7.png","title":"Docker with container ID, Image, Command, and Created"}]}],["$","p",null,{"children":["You can now browse to ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"http://localhost:9000/#/zookeeper","children":"http://localhost:9000/#/zookeeper"}]," to see the running cluster:"]}],["$","p",null,{"children":["$","img",null,{"alt":"Empty Zookeeper Browser","src":"https://www.datocms-assets.com/75153/1685462203-image5.png","title":"Empty Zookeeper Browser"}]}],["$","h3",null,{"id":"step-2-install-kafka-on-the-docker-container","children":[["$","a",null,{"href":"#step-2-install-kafka-on-the-docker-container","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Step 2: Install Kafka on the Docker container"]}],["$","p",null,{"children":"Next, let's install Kafka. We will be installing Kafka on the existing docker container. For this step, download the TAR file, extract the contents, and start Kafka."}],["$","p",null,{"children":["$","em",null,{"children":"Apache Kafka is an open source software platform that provides a unified, high-throughput, low-latency platform for handling real-time data feeds."}]}],["$","p",null,{"children":"Use the following command to download the Kafka image:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token class-name builtin","children":"cd"}]," ",["$","span",null,{"className":"token punctuation","children":".."}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"curl"}]," https://downloads.apache.org/kafka/3.4.0/kafka_2.12-3.4.0.tgz ",["$","span",null,{"className":"token parameter variable","children":"--output"}]," kafka.tgz ",["$","span",null,{"className":"token parameter variable","children":"--output"}]," kafka.tgz\n"]}]]}]}],["$","p",null,{"children":"It should look this:"}],["$","p",null,{"children":["$","img",null,{"alt":"Code with Apache Pinot speed results","src":"https://www.datocms-assets.com/75153/1685462322-image8.png","title":"Code with Apache Pinot speed results"}]}],["$","p",null,{"children":"Note that we’ve changed the directory to keep the Kafka folder separate from the Pinot folder."}],["$","p",null,{"children":"Now, let’s expand the downloaded TAR file, rename the folder for convenience, and delete the downloaded file:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"tar"}]," ",["$","span",null,{"className":"token parameter variable","children":"-xvf"}]," kafka.tgz\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"mv"}]," kafka_2.12-3.4.0 kafka\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"rm"}]," ",["$","span",null,{"className":"token parameter variable","children":"-rf"}]," kafka.tgz\n"]}]]}]}],["$","p",null,{"children":"It should look like this:"}],["$","p",null,{"children":["$","img",null,{"alt":"Code with Apache Kafka","src":"https://www.datocms-assets.com/75153/1685462061-image2.png","title":"Code with Apache Kafka"}]}],["$","p",null,{"children":["$","img",null,{"alt":"Code with kafka version","src":"https://www.datocms-assets.com/75153/1685462480-image12.png","title":"Code with kafka version"}]}],["$","p",null,{"children":"Now, Kafka and Pinot reside locally on our Docker container with Pinot up and running. Let’s run the Kafka service. Kafka will use the existing ZooKeeper for configuration management."}],["$","p",null,{"children":"Use the following command to run Kafka:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token class-name builtin","children":"cd"}]," kafka\n"]}],["$","span",null,{"className":"code-line","children":"./bin/kafka-server-start.sh config/server.properties\n"}]]}]}],["$","p",null,{"children":"It should look like this:"}],["$","p",null,{"children":["$","img",null,{"alt":"Code with cd kafka","src":"https://www.datocms-assets.com/75153/1685462400-image10.png","title":"Code with cd kafka"}]}],["$","p",null,{"children":["To verify that Kafka is running, let’s look at our ZooKeeper configs by browsing to ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"http://localhost:9000/#/zookeeper","children":"http://localhost:9000/#/zookeeper:"}]]}],["$","p",null,{"children":["$","img",null,{"alt":"Zookeeper Browser","src":"https://www.datocms-assets.com/75153/1685462099-image3.png","title":"Zookeeper Browser"}]}],["$","p",null,{"children":"You may have to refresh the page and find many more configuration items appear thanexpectedt. These are Kafka configurations."}],["$","h3",null,{"id":"step-3-ingest-data-into-kafka","children":[["$","a",null,{"href":"#step-3-ingest-data-into-kafka","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Step 3: Ingest data into Kafka"]}],["$","p",null,{"children":"In this step, we will ingest data into Kafka. We will be using Wikipedia events since they are easily accessible. We will use a node script to ingest the Wikipedia events, then add them to a Kafka Topic."}],["$","p",null,{"children":"Let’s first create some folders like this:"}],["$","p",null,{"children":"cd /opt"}],["$","p",null,{"children":"mkdir realtime"}],["$","p",null,{"children":"cd realtime"}],["$","p",null,{"children":"mkdir events"}],["$","p",null,{"children":"It should look like this:"}],["$","p",null,{"children":["$","img",null,{"alt":"Code with realtime","src":"https://www.datocms-assets.com/75153/1685462150-image4.png","title":"Code with realtime"}]}],["$","p",null,{"children":"You may have to start a new PowerShell window and connect to Docker for this. Now, let’s install Node.js and any dependencies we might need for the event consumption script:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"curl"}]," ",["$","span",null,{"className":"token parameter variable","children":"-fsSL"}]," https://deb.nodesource.com/setup_14.x ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token function","children":"bash"}]," -\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"apt"}]," ",["$","span",null,{"className":"token function","children":"install"}]," nodejs\n"]}]]}]}],["$","p",null,{"children":"Node.js takes a few minutes to install. Next, we will create a script to consume the events called wikievents.js. Cut and paste the following code to this file:"}],["$","$L11",null,{"className":"language-javascript","children":["$","code",null,{"className":"code-highlight language-javascript","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"var"}]," ",["$","span",null,{"className":"token maybe-class-name","children":"EventSource"}]," ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token function","children":"require"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token string","children":"'eventsource'"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"var"}]," fs ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token function","children":"require"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token string","children":"'fs'"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"var"}]," path ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token function","children":"require"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token string","children":"'path'"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"const"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}]," ",["$","span",null,{"className":"token maybe-class-name","children":"Kafka"}]," ",["$","span",null,{"className":"token punctuation","children":"}"}]," ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token function","children":"require"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token string","children":"'kafkajs'"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"var"}]," url ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token string","children":"'https://stream.wikimedia.org/v2/stream/recentchange'"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"const"}]," kafka ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token keyword","children":"new"}]," ",["$","span",null,{"className":"token class-name","children":"Kafka"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property literal-property","children":"clientId"}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"'wikievents'"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property literal-property","children":"brokers"}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token string","children":"'localhost:9092'"}],["$","span",null,{"className":"token punctuation","children":"]"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"const"}]," producer ",["$","span",null,{"className":"token operator","children":"="}]," kafka",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"producer"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"async"}]," ",["$","span",null,{"className":"token keyword","children":"function"}]," ",["$","span",null,{"className":"token function","children":"start"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token keyword control-flow","children":"await"}]," producer",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"connect"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token function","children":"startEvents"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"function"}]," ",["$","span",null,{"className":"token function","children":"startEvents"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token class-name console","children":"console"}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"log"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token template-string","children":[["$","span",null,{"className":"token string template-punctuation","children":"`"}],["$","span",null,{"className":"token string","children":"Connecting to EventStreams at "}],["$","span",null,{"className":"token interpolation","children":[["$","span",null,{"className":"token punctuation interpolation-punctuation","children":"$${"}],"url",["$","span",null,{"className":"token punctuation interpolation-punctuation","children":"}"}]]}],["$","span",null,{"className":"token string template-punctuation","children":"`"}]]}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token keyword","children":"var"}]," eventSource ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token keyword","children":"new"}]," ",["$","span",null,{"className":"token class-name","children":"EventSource"}],["$","span",null,{"className":"token punctuation","children":"("}],"url",["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":["    eventSource",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method function-variable method-variable","children":"onopen"}]," ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token keyword","children":"function"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token class-name console","children":"console"}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"log"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token string","children":"'--- Opened connection.'"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":["    eventSource",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method function-variable method-variable","children":"onerror"}]," ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token keyword","children":"function"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token parameter","children":"event"}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token class-name console","children":"console"}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"error"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token string","children":"'--- Encountered error'"}],["$","span",null,{"className":"token punctuation","children":","}]," event",["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":["    eventSource",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method function-variable method-variable","children":"onmessage"}]," ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token keyword","children":"async"}]," ",["$","span",null,{"className":"token keyword","children":"function"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token parameter","children":"event"}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token keyword","children":"const"}]," data ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token class-name known-class-name","children":"JSON"}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"parse"}],["$","span",null,{"className":"token punctuation","children":"("}],"event",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token property-access","children":"data"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token keyword","children":"const"}]," eventPath ",["$","span",null,{"className":"token operator","children":"="}]," path",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"join"}],["$","span",null,{"className":"token punctuation","children":"("}],"__dirname",["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token string","children":"'./events'"}],["$","span",null,{"className":"token punctuation","children":","}]," data",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token property-access","children":"wiki"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        fs",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"existsSync"}],["$","span",null,{"className":"token punctuation","children":"("}],"eventPath",["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token operator","children":"||"}]," fs",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"mkdirSync"}],["$","span",null,{"className":"token punctuation","children":"("}],"eventPath",["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        fs",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"writeFileSync"}],["$","span",null,{"className":"token punctuation","children":"("}],"path",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"join"}],["$","span",null,{"className":"token punctuation","children":"("}],"eventPath",["$","span",null,{"className":"token punctuation","children":","}]," data",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token property-access","children":"meta"}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token property-access","children":"id"}]," ",["$","span",null,{"className":"token operator","children":"+"}]," ",["$","span",null,{"className":"token string","children":"'.json'"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}]," event",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token property-access","children":"data"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token keyword control-flow","children":"await"}]," producer",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token function property-access method","children":"send"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property literal-property","children":"topic"}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"'wikipedia-events'"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property literal-property","children":"messages"}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["                    ",["$","span",null,{"className":"token property literal-property","children":"key"}],["$","span",null,{"className":"token operator","children":":"}]," data",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token property-access","children":"meta"}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token property-access","children":"id"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["                    ",["$","span",null,{"className":"token property literal-property","children":"value"}],["$","span",null,{"className":"token operator","children":":"}]," event",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token property-access","children":"data"}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"]"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"start"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":";"}],"\n"]}]]}]}],["$","p",null,{"children":"You can use vi to create the file and save it. You can also use Docker Desktop to edit the file."}],["$","p",null,{"children":"To install the two modules referenced in the file above, kafkajs and eventsource, run the following command:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"npm"}]," i eventsource kafkajs\n"]}]}]}],["$","p",null,{"children":"Let’s run the program. This will result in the download of many files, so I recommend running the program for just a few minutes. You can stop the run by using Ctrl-C."}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"node"}]," wikievents.js\n"]}]}]}],["$","p",null,{"children":"Use Ctrl-C to stop the program. Navigate to the events folder to see some new folders created with the various language events downloaded from Wikipedia."}],["$","p",null,{"children":["$","img",null,{"alt":"Wikievents node in code","src":"https://www.datocms-assets.com/75153/1685462366-image9.png","title":"Wikievents node in code"}]}],["$","p",null,{"children":"Navigate to the enwiki folder and review some of the downloaded JSON files."}],["$","p",null,{"children":["$","img",null,{"alt":"Code with realtime wikievents","src":"https://www.datocms-assets.com/75153/1685462441-image11.png","title":"Code with realtime wikievents"}]}],["$","p",null,{"children":["At ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"http://localhost:9000/#/zookeeper","children":"http://localhost:9000/#/zookeeper"}],", you can find the Kafka topic by locating the ZooKeeper config and expanding config > topics. You may have to refresh your browser."]}],["$","p",null,{"children":["$","img",null,{"alt":"Zookeeper browser in Apache Pinot topics","src":"https://www.datocms-assets.com/75153/1685462510-image13.png","title":"Zookeeper browser in Apache Pinot topics"}]}],["$","p",null,{"children":"Here, you should see the wikipedia-events topic that we created using the Node.js script. So far, so good."}],["$","h3",null,{"id":"step-4-connect-kafka-to-pinot","children":[["$","a",null,{"href":"#step-4-connect-kafka-to-pinot","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Step 4: Connect Kafka to Pinot"]}],["$","p",null,{"children":"With Kafka installed and configured to receive events, we can connect it to Pinot."}],["$","p",null,{"children":"To create a real-time table in Pinot that can consume the Kafka topic, create a schema and a configuration table. The schema configuration is very much like the schema that we created for our batch example. You can use vi to create a file named realtime.schema.json and cut and paste the content below."}],["$","p",null,{"children":"Here’s the JSON for the wikievents schema:"}],["$","$L11",null,{"className":"language-json","children":["$","code",null,{"className":"code-highlight language-json","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"schemaName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"wikievents\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"dimensionFieldSpecs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"id\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"wiki\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"user\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"title\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"comment\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"stream\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"domain\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"topic\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"type\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"metaJson\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"dateTimeFieldSpecs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"timestamp\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"LONG\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"format\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"1:MILLISECONDS:EPOCH\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"granularity\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"1:MILLISECONDS\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"]"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}]]}]}],["$","p",null,{"children":"Creating the table config file is where the magic happens. Use vi (or your favorite editor) to create realtime.tableconfig.json and cut and paste the following content:"}],["$","$L11",null,{"className":"language-json","children":["$","code",null,{"className":"code-highlight language-json","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"tableName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"wikievents_REALTIME\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"tableType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"REALTIME\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"segmentsConfig\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"timeColumnName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"timestamp\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"schemaName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"wikievents\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"replication\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"1\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"replicasPerPartition\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"1\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"tenants\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"broker\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"DefaultTenant\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"server\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"DefaultTenant\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"tagOverrideConfig\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"tableIndexConfig\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"invertedIndexColumns\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"rangeIndexColumns\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"autoGeneratedInvertedIndex\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token boolean","children":"false"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"createInvertedIndexDuringSegmentGeneration\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token boolean","children":"false"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"sortedColumn\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"bloomFilterColumns\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"loadMode\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"MMAP\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"streamConfigs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"streamType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"kafka\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"stream.kafka.topic.name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"wikipedia-events\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"stream.kafka.broker.list\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"localhost:9092\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"stream.kafka.consumer.type\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"lowlevel\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"stream.kafka.consumer.prop.auto.offset.reset\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"smallest\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"stream.kafka.consumer.factory.class.name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"stream.kafka.decoder.class.name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"realtime.segment.flush.threshold.rows\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"0\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"realtime.segment.flush.threshold.time\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"24h\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token property","children":"\"realtime.segment.flush.segment.size\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"100M\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"noDictionaryColumns\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"onHeapDictionaryColumns\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"varLengthDictionaryColumns\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"enableDefaultStarTree\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token boolean","children":"false"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"enableDynamicStarTreeCreation\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token boolean","children":"false"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"aggregateMetrics\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token boolean","children":"false"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"nullHandlingEnabled\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token boolean","children":"false"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"metadata\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"quota\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"routing\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"query\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"ingestionConfig\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token property","children":"\"transformConfigs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"columnName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"metaJson\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"transformFunction\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"JSONFORMAT(meta)\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"columnName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"id\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"transformFunction\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"JSONPATH(metaJson, '$.id')\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"columnName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"stream\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"transformFunction\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"JSONPATH(metaJson, '$.stream')\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"columnName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"domain\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"transformFunction\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"JSONPATH(metaJson, '$.domain')\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"columnName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"topic\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["                ",["$","span",null,{"className":"token property","children":"\"transformFunction\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"JSONPATH(metaJson, '$.topic')\""}],"\n"]}],["$","span",null,{"className":"code-line","children":["            ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":["        ",["$","span",null,{"className":"token punctuation","children":"]"}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":["    ",["$","span",null,{"className":"token property","children":"\"isDimTable\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token boolean","children":"false"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}]]}]}],["$","p",null,{"children":"Notice the section called streamConfigs, where we define the source as a Kafka stream, located at localhost:9092, and consume the topic wikipedia-events. That’s all it takes to consume a Kafka Topic into Pinot."}],["$","p",null,{"children":"Don’t believe me? Give it a try!"}],["$","p",null,{"children":"Create the table by running the following command:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"language-bash code-highlight","children":["$","span",null,{"className":"code-line","children":["/opt/pinot/bin/pinot-admin.sh AddTable ",["$","span",null,{"className":"token parameter variable","children":"-schemaFile"}]," /opt/realtime/realtime.schema.json ",["$","span",null,{"className":"token parameter variable","children":"-tableConfigFile"}]," /opt/realtime/realtime.tableconfig.json ",["$","span",null,{"className":"token parameter variable","children":"-exec"}],"\n"]}]}]}],["$","p",null,{"children":["Now, browse to the following location ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"http://localhost:9000/#/tables","children":"http://localhost:9000/#/tables,"}]," and you should see the newly created table. However, where’s the real-time data, you say?"]}],["$","p",null,{"children":"Run the node wikievents.js command, then query the newly created wikievents table to see the totalDocs increase in real time:"}],["$","p",null,{"children":["$","img",null,{"alt":"Apache Pinot query console","src":"https://www.datocms-assets.com/75153/1685462248-image6.png","title":"Apache Pinot query console"}]}],["$","p",null,{"children":"To avoid running out of space on your computer, make sure to stop the wikievents.js script when you’re done :-D"}],["$","h2",null,{"id":"conclusion","children":[["$","a",null,{"href":"#conclusion","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Conclusion"]}],["$","p",null,{"children":"Congratulations! Using only the table config, we simultaneously consumed Kafka topics directly into Pinot tables and queried events. We also transformed JSON to map to the Pinot table. In the transformConfigs portion of the Pinot table config file, we consumed the nested block meta into a field called metaJson. In the subsequent steps, we referenced the metaJson field with jsonPath to extract fields such as id, stream, domain, and topic."}],["$","p",null,{"children":"Not only does Pinot support easy ingestion from Kafka topics, but it also provides a robust way to transform JSON to OLAP tables."}],["$","p",null,{"children":"In summary, we have:"}],["$","ul",null,{"children":[["$","li",null,{"children":"Installed and run Kafka"}],["$","li",null,{"children":"Consumed events from Wikipedia into Kafka"}],["$","li",null,{"children":"Created a real-time table schema and a table in Pinot"}],["$","li",null,{"children":"Streamed events from Wikipedia into Pinot tables via Kafka topics"}],["$","li",null,{"children":"Run multiple queries"}],["$","li",null,{"children":"Performed JSON transformations"}]]}],["$","p",null,{"children":["In some upcoming blog posts, we will explore more advanced topics, such as indexes and transformations, not to mention real-time anomaly detection with ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://dev.startree.ai/docs/procedures/get-started-with-thirdeye/","children":"ThirdEye"}],"."]}],["$","p",null,{"children":["In the meantime, run more queries, load more data, and don’t forget to ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://dev.startree.ai/slack-invite","children":"join the community Slack for support"}]," if you get stuck or would like to request a topic for me to write about—you know where to find us!"]}]]}]}]}]]}],["$","aside",null,{"className":"mt-10 hidden border-l-2 pl-5 lg:sticky lg:top-1 lg:block lg:h-full","children":["$","section",null,{"className":"sticky top-0 mb-4 w-[15.375rem]","children":[["$","div",null,{"className":"flex flex-col space-y-1.5 pb-3","children":["$","h3",null,{"className":"text-sm font-semibold leading-snug text-neutral-500 dark:text-neutral-100","children":"Table of Contents"}]}],["$","$L12",null,{"chapters":[{"value":"The obligatory “What is Apache Pinot and StarTree?” section","url":"#the-obligatory-what-is-apache-pinot-and-startree-section","depth":3},{"value":"How to install Kafka alongside Pinot ","url":"#how-to-install-kafka-alongside-pinot","depth":2},{"value":"Prerequisite","url":"#prerequisite","depth":3},{"value":"Step 1: Install Kafka on your Pinot Docker image","url":"#step-1-install-kafka-on-your-pinot-docker-image","depth":3},{"value":"Step 2: Install Kafka on the Docker container","url":"#step-2-install-kafka-on-the-docker-container","depth":3},{"value":"Step 3: Ingest data into Kafka","url":"#step-3-ingest-data-into-kafka","depth":3},{"value":"Step 4: Connect Kafka to Pinot","url":"#step-4-connect-kafka-to-pinot","depth":3},{"value":"Conclusion","url":"#conclusion","depth":2}]}]]}]}]]}]}]]}]]
d:null