{"version":1,"pages":[{"id":"fv9U5LYQY9sLFsvBs7Wb","title":"Data Pipeline","pathname":"/data-pipeline-2","siteSpaceId":"sitesp_uyO3L","description":"Data pipelines are used to ingest and transfer data from different sources, transform unify and cleanse so that it’s suitable for analytics and business reporting."},{"id":"aFB4reuJbZEWreHzSfKW","title":"About Data Pipeline","pathname":"/data-pipeline-2/data-pipeline/about-data-pipeline","siteSpaceId":"sitesp_uyO3L","description":"The BDB Data Pipeline is an Enterprise event based data orchestration and transformation tool, that allows you to seamlessly design and deploy your DataOPs or MLOPs workflow.","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"JFKMt4xBysCgIXX7t8S2","title":"Design Philosophy","pathname":"/data-pipeline-2/data-pipeline/design-philosophy","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"tXGk1jNlYfqYbQ8CXl7b","title":"Low Code Visual Authoring","pathname":"/data-pipeline-2/data-pipeline/low-code-visual-authoring","siteSpaceId":"sitesp_uyO3L","description":"Assembling a data pipeline is very simple. Just click and drag the component you want to use into editor canvas. Connect the component output to an event/topic.","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"kMPv1Wj6sdjZiMxkCgiv","title":"Real-time and Batch Orchestration","pathname":"/data-pipeline-2/data-pipeline/real-time-and-batch-orchestration","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"xIVUrGL1Y5Rp8oKSfwKO","title":"Event based Process Orchestration","pathname":"/data-pipeline-2/data-pipeline/event-based-process-orchestration","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"PbA8k7Bbiy7gG7sjcHD9","title":"ML and Data Ops","pathname":"/data-pipeline-2/data-pipeline/ml-and-data-ops","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"8mRDqdYjIzyYJfVbMQgw","title":"Distributed Compute","pathname":"/data-pipeline-2/data-pipeline/distributed-compute","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"00rUqpfF7gOSWAYqf938","title":"Fault Tolerant and Auto-recovery","pathname":"/data-pipeline-2/data-pipeline/fault-tolerant-and-auto-recovery","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"744YfD4HZtO5UO2wrMdo","title":"Extensibility via Custom Scripting","pathname":"/data-pipeline-2/data-pipeline/extensibility-via-custom-scripting","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Data Pipeline"}]},{"id":"unzFHfsgWR83EpPR0juu","title":"Getting Started","pathname":"/data-pipeline-2/getting-started","siteSpaceId":"sitesp_uyO3L"},{"id":"YBwd7RIl1LFZifi478ZK","title":"Homepage","pathname":"/data-pipeline-2/getting-started/homepage","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"}]},{"id":"scfXSOZkM3q2cBleNRUm","title":"List Pipelines","pathname":"/data-pipeline-2/getting-started/homepage/list-pipelines","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"cfInHXlvP6oeAtTEccnb","title":"Create","pathname":"/data-pipeline-2/getting-started/homepage/create","siteSpaceId":"sitesp_uyO3L","description":"Create a new Pipeline or Job based on the selected create option.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"hbMHNy1wKgInTFlW77Nh","title":"Creating a New Pipeline","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-pipeline","siteSpaceId":"sitesp_uyO3L","description":"This section provides the steps involved in creating a new Pipeline flow.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"}]},{"id":"lNDvU5MJNFES84aayLNd","title":"Adding Components to Canvas","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-pipeline/adding-components-to-canvas","siteSpaceId":"sitesp_uyO3L","description":"Adding components to a pipeline workflow.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Pipeline"}]},{"id":"3qQRf7OOIt2sepDpfxfs","title":"Connecting Components","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-pipeline/connecting-components","siteSpaceId":"sitesp_uyO3L","description":"An event-driven architecture uses events to trigger and communicate between decoupled services and is common in modern applications built with microservices.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Pipeline"}]},{"id":"9zUXqQWafz2HJWAVimcU","title":"Events [Kafka and Data Sync]","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-pipeline/connecting-components/events-kafka-and-data-sync","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Pipeline"},{"label":"Connecting Components"}]},{"id":"FHG5HD5UlzyWmtuU58sq","title":"Memory and CPU Allocations","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-pipeline/memory-and-cpu-allocations","siteSpaceId":"sitesp_uyO3L","description":"This section focuses on the Configuration tab provided for any Pipeline component.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Pipeline"}]},{"id":"uZxIIjnDL7WNiSp2TWdX","title":"Creating a New Job","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job","siteSpaceId":"sitesp_uyO3L","description":"This section provides detailed information on the Jobs to make your data process faster.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"}]},{"id":"uH97oegAMterbRRexoL6","title":"Job Editor Page","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/job-editor-page","siteSpaceId":"sitesp_uyO3L","description":"The Job Editor Page provides the user with all the necessary options and components to add a task and eventually create a Job workflow.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"}]},{"id":"Y0JuExqxOkl0ihcUFPCT","title":"Task Components","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components","siteSpaceId":"sitesp_uyO3L","description":"This Section provides you the details about the various categories of the Task components included inside the Job Editor page.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"}]},{"id":"sTaViK4oknRZqOshmS9n","title":"Readers","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/readers","siteSpaceId":"sitesp_uyO3L","description":"All the available Reader Task components are included in this section.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"}]},{"id":"O9guJzJb2dC5gcvE093i","title":"HDFS Reader","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/readers/hdfs-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Readers"}]},{"id":"YYKL4dPAiB5FD2B4x647","title":"MongoDB Reader","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/readers/mongodb-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Readers"}]},{"id":"vQHpIyMv3RhQTwCARHbm","title":"DB Reader","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/readers/db-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Readers"}]},{"id":"EtJ2fLLlRX2sAWMB3fl4","title":"S3 Reader","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/readers/s3-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Readers"}]},{"id":"f8C7WWpvAunYYYrk8OYv","title":"Azure Blob Reader","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/readers/azure-blob-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Readers"}]},{"id":"X5qefvDOMEUeb2ftIFBY","title":"ES Reader","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/readers/es-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Readers"}]},{"id":"mUwnHuVdF11Fx0536Efy","title":"Sandbox Reader","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/readers/sandbox-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Readers"}]},{"id":"5SAW1fPmT9TuhfFcooqb","title":"Writers","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers","siteSpaceId":"sitesp_uyO3L","description":"All the available Writer Task components for a Job are explained in this section.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"}]},{"id":"LoT8MlN5zqkhJMuNuZx8","title":"HDFS Writer","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers/hdfs-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Writers"}]},{"id":"QVd6nzuunoPiA0zS9kMu","title":"Azure Writer","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers/azure-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Writers"}]},{"id":"4xviEjIYxGqGZMjkI8Ks","title":"DB Writer","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers/db-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Writers"}]},{"id":"AJR7cHyyCGGLIenl0ZnF","title":"ES Writer","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers/es-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Writers"}]},{"id":"r2JYMIa7n9amMZ8CLz5v","title":"S3 Writer","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers/s3-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Writers"}]},{"id":"lCVJNspKw6KdioQQCn42","title":"Sandbox Writer","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers/sandbox-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Writers"}]},{"id":"ajDOCBhhMcs7P6QwSazi","title":"Mongodb Writer","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers/mongodb-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Writers"}]},{"id":"wNNft8eAxoehpjrlH44b","title":"Kafka Producer","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/writers/kafka-producer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"},{"label":"Writers"}]},{"id":"ShZOjAEgBqnhwAiZ0ebc","title":"Transformations","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/task-components/transformations","siteSpaceId":"sitesp_uyO3L","description":"This page aims to explain the various transformation options provided on the Jobs Editor page.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"},{"label":"Task Components"}]},{"id":"vpx0DKImWiUNc1YaQ4tT","title":"PySpark Job","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/pyspark-job","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"}]},{"id":"lztvKpTaByRjtx0rjVIg","title":"Python Job","pathname":"/data-pipeline-2/getting-started/homepage/create/creating-a-new-job/python-job","siteSpaceId":"sitesp_uyO3L","description":"Write Python scripts and run them flawlessly in the Jobs.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"},{"label":"Create"},{"label":"Creating a New Job"}]},{"id":"iXWUEQQPy5ruAInLFmZR","title":"List Jobs","pathname":"/data-pipeline-2/getting-started/homepage/list-jobs","siteSpaceId":"sitesp_uyO3L","description":"All the saved Jobs by a logged-in user get listed by using this option.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"KRKCRFs5Wj8TzwxJ2cSa","title":"List Components","pathname":"/data-pipeline-2/getting-started/homepage/list-components","siteSpaceId":"sitesp_uyO3L","description":"All the components that a user can use in their pipeline, are listed here. It has a complete info like deployment Type, versions etc.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"IDgBqQS211YDIqJqbb9S","title":"Delete Orphan Pods","pathname":"/data-pipeline-2/getting-started/homepage/delete-orphan-pods","siteSpaceId":"sitesp_uyO3L","description":"This is to delete the pods that is no more a part of the current Pipelines.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"9cf0PHdZCWdJ8RYXDktW","title":"Scheduler","pathname":"/data-pipeline-2/getting-started/homepage/scheduler","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"mTYGRBmDfsALCCXCCQmW","title":"Data Channel","pathname":"/data-pipeline-2/getting-started/homepage/data-channel","siteSpaceId":"sitesp_uyO3L","description":"Data Chanel shows the health of the Kafka Brokers.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"jH1XkcDWwS9r60ZvUnHw","title":"Cluster Event","pathname":"/data-pipeline-2/getting-started/homepage/cluster-event","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"MyRvPBZxQbvXflXAQEct","title":"Trash","pathname":"/data-pipeline-2/getting-started/homepage/trash","siteSpaceId":"sitesp_uyO3L","description":"This page describe how to delete an pipeline or restore a deleted pipeline using the Trash option.","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"eEpGxqxvDbSHOR7AsFOH","title":"Settings","pathname":"/data-pipeline-2/getting-started/homepage/settings","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Homepage"}]},{"id":"alkxnFTSz3ckUlPh1nWa","title":"Pipeline Workflow Editor","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor","siteSpaceId":"sitesp_uyO3L","description":"The Pipeline Workflow Editor contains Toolbar, Component Panel, and the Right-side Panel together with the Design Canvas for the user to create a pipeline flow.","breadcrumbs":[{"label":"Getting Started"}]},{"id":"w6fhcISLkT5vqgaeB8H1","title":"Pipeline Toolbar","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar","siteSpaceId":"sitesp_uyO3L","description":"The Header menu panel on the Pipeline Workflow Editor page.","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"}]},{"id":"uYZu7dPC7XicaPtnZlOT","title":"Pipeline Overview","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/pipeline-overview","siteSpaceId":"sitesp_uyO3L","description":"This page provides an overview of all the components used in the pipeline in a single place.","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"subkBYmrOm5U16T1xNyj","title":"Pipeline Testing","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/pipeline-testing","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"kN8cHYRkHFhCEqpJJ5ww","title":"Search Component in Pipelines","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/search-component-in-pipelines","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"dwBbRYsjFiQf0ziAkHd7","title":"Push Pipeline (to VCS/GIT)","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/push-pipeline-to-vcs-git","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"5dyXp32wFZDU55DYY7fa","title":"Pull Pipeline","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/pull-pipeline","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"zzjsXjTUVt8db7HkVGGz","title":"Full Screen","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/full-screen","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"ZW9ciFOUe1WdJ7xucqeY","title":"Log Panel","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/log-panel","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"kbUdCLv7zrRGIBaBjZHO","title":"Event Panel","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/event-panel","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"gktCqlYvmVf5ROlKcMqm","title":"Activate/Deactivate Pipeline","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/activate-deactivate-pipeline","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"pnnIjJxLNL34AQJOBAk4","title":"Update Pipeline","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/update-pipeline","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"aFjXHmz426eNX1XeLBy1","title":"Failure Analysis","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/failure-analysis","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"tetfoG7Ey6r6h3PrdvlH","title":"Pipeline Monitoring","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/pipeline-monitoring","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"8gKYVOcp5nJptWoh1A9l","title":"Delete Pipeline","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/delete-pipeline","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"W7nB4IVp7MxswNnpG9mX","title":"Pipeline Component Configuration","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/pipeline-component-configuration","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"Eth3WBie76btAK8RGfFn","title":"Pipeline Failure Alert History","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/pipeline-toolbar/pipeline-failure-alert-history","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"},{"label":"Pipeline Toolbar"}]},{"id":"SeTSjvbD4IGkzEEdfiKZ","title":"Component Panel","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/component-panel","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"}]},{"id":"oj7QqSgYEK6uKs6pEyjo","title":"Right-side Panel","pathname":"/data-pipeline-2/getting-started/pipeline-workflow-editor/right-side-panel","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"},{"label":"Pipeline Workflow Editor"}]},{"id":"nhcGHpgX19piu6o5f1aY","title":"Testing Suite","pathname":"/data-pipeline-2/getting-started/testing-suite","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Getting Started"}]},{"id":"ysOCx8Io35E2JbIHSw7c","title":"Activating Pipeline","pathname":"/data-pipeline-2/getting-started/activating-pipeline","siteSpaceId":"sitesp_uyO3L","description":"Activating pipeline to deploy the pipelines.","breadcrumbs":[{"label":"Getting Started"}]},{"id":"I8n1AwkqjZHxprk5RVFN","title":"Monitoring Pipeline","pathname":"/data-pipeline-2/getting-started/monitoring-pipeline","siteSpaceId":"sitesp_uyO3L","description":"This Page explains How we can monitor the Pipelines.","breadcrumbs":[{"label":"Getting Started"}]},{"id":"af1z2x29qDsEKSTsWrz3","title":"Job Monitoring","pathname":"/data-pipeline-2/getting-started/job-monitoring","siteSpaceId":"sitesp_uyO3L","description":"This page explains how we can monitor a Job.","breadcrumbs":[{"label":"Getting Started"}]},{"id":"Eb5ZsSAUAKi2g052l3Lx","title":"Components","pathname":"/data-pipeline-2/components","siteSpaceId":"sitesp_uyO3L"},{"id":"oc62hDHkStJPNoyhEXOr","title":"Adding Components to Workflow","pathname":"/data-pipeline-2/components/adding-components-to-workflow","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"}]},{"id":"TQy1A1ue2uPEF2gWN9IW","title":"Component Architecture","pathname":"/data-pipeline-2/components/component-architecture","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"}]},{"id":"ErW9nlvv0A2XKK5QwHZW","title":"Component Base Configuration","pathname":"/data-pipeline-2/components/component-base-configuration","siteSpaceId":"sitesp_uyO3L","description":"This page pays attention to describe the Basic Info tab provided for the pipeline components. This tab has to be configured for all the components.","breadcrumbs":[{"label":"Components"}]},{"id":"sisel1kEJGqCR9WJDLhQ","title":"Resource Configuration","pathname":"/data-pipeline-2/components/resource-configuration","siteSpaceId":"sitesp_uyO3L","description":"There is a resource configuration tab while configuring the components.","breadcrumbs":[{"label":"Components"}]},{"id":"1dI4Dz7SS1HwMq0ctxYx","title":"Intelligent Scaling","pathname":"/data-pipeline-2/components/intelligent-scaling","siteSpaceId":"sitesp_uyO3L","description":"A way to scale up the processing speed of components.","breadcrumbs":[{"label":"Components"}]},{"id":"jmlJxSkJqvPofMCOruvw","title":"Connection Validation","pathname":"/data-pipeline-2/components/connection-validation","siteSpaceId":"sitesp_uyO3L","description":"The Connection Validation option helps the users to validate the connection details of the db/cloud storages.","breadcrumbs":[{"label":"Components"}]},{"id":"Vqk8Hv0xXihEOrxnJd4o","title":"Readers","pathname":"/data-pipeline-2/components/readers","siteSpaceId":"sitesp_uyO3L","description":"Readers are a group of components that can read data from different DB and cloud storages in both invocation types i.e., Real-Time and Batch.","breadcrumbs":[{"label":"Components"}]},{"id":"i7SbvV5sOhuOBoptUn3N","title":"S3 Reader","pathname":"/data-pipeline-2/components/readers/s3-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"GdGz5ofWyrrAFjrO1mLU","title":"HDFS Reader","pathname":"/data-pipeline-2/components/readers/hdfs-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"sfpn1rldSCxFvqBKUI47","title":"DB Reader","pathname":"/data-pipeline-2/components/readers/db-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"TvBhepGbMIQZve7D6X0S","title":"ES Reader","pathname":"/data-pipeline-2/components/readers/es-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"qwrMyj0bnWBZh0t8Eymz","title":"SFTP Stream Reader","pathname":"/data-pipeline-2/components/readers/sftp-stream-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"d9OxmES7d8IWEphHA7lT","title":"SFTP Reader","pathname":"/data-pipeline-2/components/readers/sftp-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"yewerOy6z0oy1nHWHy6f","title":"Mongo DB Reader","pathname":"/data-pipeline-2/components/readers/mongo-db-reader","siteSpaceId":"sitesp_uyO3L","description":"Mongo DB reader component contains both the deployment-types: Spark & Docker","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"NHgCoCrMufIJBQBufsY1","title":"MongoDB Reader Lite (PyMongo Reader)","pathname":"/data-pipeline-2/components/readers/mongo-db-reader/mongodb-reader-lite-pymongo-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"},{"label":"Mongo DB Reader"}]},{"id":"3f9ijoNVKfLpR1asySC4","title":"MongoDB Reader","pathname":"/data-pipeline-2/components/readers/mongo-db-reader/mongodb-reader","siteSpaceId":"sitesp_uyO3L","description":"This page covers configuration details for the MongoDB Reader component.","breadcrumbs":[{"label":"Components"},{"label":"Readers"},{"label":"Mongo DB Reader"}]},{"id":"sBlHt5Ig3ds0zGKS1PGo","title":"Azure Blob Reader","pathname":"/data-pipeline-2/components/readers/azure-blob-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"6b0leWuDvlwIL9EYpWhZ","title":"Azure Metadata Reader","pathname":"/data-pipeline-2/components/readers/azure-metadata-reader","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"vzeJN3Tg7MMur9o7o89k","title":"ClickHouse Reader (Docker)","pathname":"/data-pipeline-2/components/readers/clickhouse-reader-docker","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"7QvJSvuikXgahIcXYyVw","title":"Sandbox Reader","pathname":"/data-pipeline-2/components/readers/sandbox-reader","siteSpaceId":"sitesp_uyO3L","description":"A Sandbox reader is used to read and access data within a configured sandbox environment.","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"kfA6nvFy8gPPD1WencU3","title":"Azure Blob Reader","pathname":"/data-pipeline-2/components/readers/azure-blob-reader-1","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Readers"}]},{"id":"7yRd6JyHjaey29cJZzHE","title":"Writers","pathname":"/data-pipeline-2/components/writers","siteSpaceId":"sitesp_uyO3L","description":"Data writers specifically focus on the final stage of the pipeline, where the processed or transformed data is written to the target destination. This section explains all the supported Data Writers.","breadcrumbs":[{"label":"Components"}]},{"id":"Gj4WGTk73UzkmUXy3Ovy","title":"S3 Writer","pathname":"/data-pipeline-2/components/writers/s3-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"OsokKcbezmMObFbGGiFb","title":"DB Writer","pathname":"/data-pipeline-2/components/writers/db-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"LntVBR2pHI5wxTax0PSK","title":"HDFS Writer","pathname":"/data-pipeline-2/components/writers/hdfs-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"KY8ORinM0Teu50DAthZ5","title":"ES Writer","pathname":"/data-pipeline-2/components/writers/es-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"Bt6sCaU5aJMtzskYmKOw","title":"Video Writer","pathname":"/data-pipeline-2/components/writers/video-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"b0uvZTxaLQyGLhabKxNl","title":"Azure Writer","pathname":"/data-pipeline-2/components/writers/azure-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"ta7R8bLCvuoepcrojrq9","title":"ClickHouse Writer (Docker)","pathname":"/data-pipeline-2/components/writers/clickhouse-writer-docker","siteSpaceId":"sitesp_uyO3L","description":"Along with the Spark Driver in RDBMS Writer we have Docker writer that supports TCP port.","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"RPgraQjuG5QhMMa5BkAq","title":"Sandbox Writer","pathname":"/data-pipeline-2/components/writers/sandbox-writer","siteSpaceId":"sitesp_uyO3L","description":"A Sandbox writer is used to writer data within a configured sandbox environment.","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"D8xfXC9ZK4MmNHAyLLca","title":"MongoDB Writers","pathname":"/data-pipeline-2/components/writers/mongodb-writers","siteSpaceId":"sitesp_uyO3L","description":"We have given two different writers for writing data to MongoDB. The available deployment types for the same are: Spark and Docker.","breadcrumbs":[{"label":"Components"},{"label":"Writers"}]},{"id":"OrzrdgT6ZkORNfkTlogg","title":"MongoDB Writer","pathname":"/data-pipeline-2/components/writers/mongodb-writers/mongodb-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Writers"},{"label":"MongoDB Writers"}]},{"id":"b3klfNh64k5iUOX3QivU","title":"MongoDB Writer Lite (PyMongo Writer)","pathname":"/data-pipeline-2/components/writers/mongodb-writers/mongodb-writer-lite-pymongo-writer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Writers"},{"label":"MongoDB Writers"}]},{"id":"GMNhLsRZm9gzKMnguF9x","title":"Machine Learning","pathname":"/data-pipeline-2/components/machine-learning","siteSpaceId":"sitesp_uyO3L","description":"These components utilize machine learning algorithms and techniques to analyze and model the data.","breadcrumbs":[{"label":"Components"}]},{"id":"kkIKkmTeIOPOrb66il9z","title":"DSLab Runner","pathname":"/data-pipeline-2/components/machine-learning/dslab-runner","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Machine Learning"}]},{"id":"Ae1JmbJkEdNLxwFFnBQ2","title":"AutoML Runner","pathname":"/data-pipeline-2/components/machine-learning/automl-runner","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Machine Learning"}]},{"id":"1NqrBaqLG4aucqXT3Ytr","title":"Consumers","pathname":"/data-pipeline-2/components/consumers","siteSpaceId":"sitesp_uyO3L","description":"These are the real-time / Streaming component that ingests data or monitor for change in data objects from different sources to the pipeline.","breadcrumbs":[{"label":"Components"}]},{"id":"MHClBKGSBfpu2WSAekJr","title":"SFTP Monitor","pathname":"/data-pipeline-2/components/consumers/sftp-monitor","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"dG3xFyCMNAHHwv5R4vlH","title":"MQTT Consumer","pathname":"/data-pipeline-2/components/consumers/mqtt-consumer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"lYCyEL0T48AJoW3hbz4U","title":"Video Stream Consumer","pathname":"/data-pipeline-2/components/consumers/video-stream-consumer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"tK2kc2fNUX5Hcz4045GV","title":"Eventhub Subscriber","pathname":"/data-pipeline-2/components/consumers/eventhub-subscriber","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"lohNn2pFW2GLsneTmB3O","title":"Twitter Scrapper","pathname":"/data-pipeline-2/components/consumers/twitter-scrapper","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"aTSTHWOTCrAAkWxGW5xh","title":"Mongo ChangeStream","pathname":"/data-pipeline-2/components/consumers/mongo-changestream","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"uBfuLEseVjdpwcyWV142","title":"Rabbit MQ Consumer","pathname":"/data-pipeline-2/components/consumers/rabbit-mq-consumer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"BCQpxpgKKfVEcG7xDOlx","title":"AWS SNS Monitor","pathname":"/data-pipeline-2/components/consumers/aws-sns-monitor","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"qwl1KcLlpjh6TPAQ9kO5","title":"Kafka Consumer","pathname":"/data-pipeline-2/components/consumers/kafka-consumer","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"Bw3TyuDO56dkIxeklx00","title":"API Ingestion and Webhook Listener","pathname":"/data-pipeline-2/components/consumers/api-ingestion-and-webhook-listener","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Consumers"}]},{"id":"fgPSaL88zba0h41n8MB7","title":"Producers","pathname":"/data-pipeline-2/components/producers","siteSpaceId":"sitesp_uyO3L","description":"The role of data producers is to ensure a continuous flow of data into the pipeline, providing the necessary raw material for subsequent processing and analysis.","breadcrumbs":[{"label":"Components"}]},{"id":"vCdlW1h5dSPbwT3ZHo43","title":"WebSocket Producer","pathname":"/data-pipeline-2/components/producers/websocket-producer","siteSpaceId":"sitesp_uyO3L","description":"A WebSocket producer component is a software component that is used to send data over a WebSocket connection.","breadcrumbs":[{"label":"Components"},{"label":"Producers"}]},{"id":"BqZjMXolo0a0OG75SlWS","title":"Eventhub Publisher","pathname":"/data-pipeline-2/components/producers/eventhub-publisher","siteSpaceId":"sitesp_uyO3L","description":"The EventHub Publisher leverages the scalability and throughput capabilities of Event Hubs to ensure efficient and reliable event transmission.","breadcrumbs":[{"label":"Components"},{"label":"Producers"}]},{"id":"mLImLbbmaPoG1Bp3KCyI","title":"EventGrid Producer","pathname":"/data-pipeline-2/components/producers/eventgrid-producer","siteSpaceId":"sitesp_uyO3L","description":"The EventHub Publisher serves as a bridge between the transformed data within the pipeline and the Azure Event Hubs service. It ensures the efficient and reliable transmission of data.","breadcrumbs":[{"label":"Components"},{"label":"Producers"}]},{"id":"Y5d9CiaTThEqoNim60mm","title":"RabbitMQ Producer","pathname":"/data-pipeline-2/components/producers/rabbitmq-producer","siteSpaceId":"sitesp_uyO3L","description":"RabbitMQ producer plays a vital role in enabling reliable message-based communication and data flow within a data pipeline.","breadcrumbs":[{"label":"Components"},{"label":"Producers"}]},{"id":"8BT7Z6WTeqCFplR0xhB7","title":"Kafka Producer","pathname":"/data-pipeline-2/components/producers/kafka-producer","siteSpaceId":"sitesp_uyO3L","description":"The Kafka producer acts as a data source within the pipeline, generating and publishing messages to Kafka for subsequent processing and consumption.","breadcrumbs":[{"label":"Components"},{"label":"Producers"}]},{"id":"0TThYHmHGqGnBVtG3NhN","title":"Synthetic Data Generator","pathname":"/data-pipeline-2/components/producers/synthetic-data-generator","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Producers"}]},{"id":"IHcSqMGyiVcq36Lfg3Ls","title":"Transformations","pathname":"/data-pipeline-2/components/transformations","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"}]},{"id":"X3t1twxcFSw7Pa1HLb5K","title":"SQL Component","pathname":"/data-pipeline-2/components/transformations/sql-component","siteSpaceId":"sitesp_uyO3L","description":"SQL transformer applies SQL operations to transform and manipulate data, providing flexibility and expressiveness in data transformations within a data pipeline.","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"qafjy40UZ85pcCXDrCng","title":"Dateprep Script Runner","pathname":"/data-pipeline-2/components/transformations/dateprep-script-runner","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"gd0A0IukuyYlQ8foru2v","title":"File Splitter","pathname":"/data-pipeline-2/components/transformations/file-splitter","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"FBy9UT2tVumsgtoYrMiK","title":"Rule Splitter","pathname":"/data-pipeline-2/components/transformations/rule-splitter","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"qZqtEGThZgydLPHDvyAd","title":"Stored Producer Runner","pathname":"/data-pipeline-2/components/transformations/stored-producer-runner","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"ZzOoJYBicfJQdUYj67dH","title":"Flatten JSON","pathname":"/data-pipeline-2/components/transformations/flatten-json","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"7iuHdQNiYTsv1Q8G3ZP3","title":"Email Component","pathname":"/data-pipeline-2/components/transformations/email-component","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"91p62qPG4DyHHEAT4R1e","title":"Pandas Query Component","pathname":"/data-pipeline-2/components/transformations/pandas-query-component","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"Rf3QuzQrhN4V92IjpiAM","title":"Enrichment Component","pathname":"/data-pipeline-2/components/transformations/enrichment-component","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"5vyyY7Bqk1CUrZx3OCy1","title":"Mongo Aggregation","pathname":"/data-pipeline-2/components/transformations/mongo-aggregation","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"La7GAFRf9kAd3zLbOeTa","title":"Data Loss Protection","pathname":"/data-pipeline-2/components/transformations/data-loss-protection","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"g2bkDsgKm941fJsi0CZU","title":"Data Preparation (Docker)","pathname":"/data-pipeline-2/components/transformations/data-preparation-docker","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"FiyePKNZcoazS9xusPBs","title":"Rest Api Component","pathname":"/data-pipeline-2/components/transformations/rest-api-component","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"SGyX2Q9t7kFmjaRBkFrs","title":"Schema Validator","pathname":"/data-pipeline-2/components/transformations/schema-validator","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Transformations"}]},{"id":"zDAHu85Sy8N8o9JboSAn","title":"Scripting","pathname":"/data-pipeline-2/components/scripting","siteSpaceId":"sitesp_uyO3L","description":"Data Pipeline module provides two types of scripting components to facilitate the users.","breadcrumbs":[{"label":"Components"}]},{"id":"947TlJJNuWSnwRRGhywR","title":"Script Runner","pathname":"/data-pipeline-2/components/scripting/script-runner","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Scripting"}]},{"id":"hzuopj8ykHGAAs29UguO","title":"Python Script","pathname":"/data-pipeline-2/components/scripting/python-script","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Scripting"}]},{"id":"qvS3H5WNONLt0pwFztMf","title":"Keeping Different Versions of the Python Script in VCS","pathname":"/data-pipeline-2/components/scripting/python-script/keeping-different-versions-of-the-python-script-in-vcs","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Components"},{"label":"Scripting"},{"label":"Python Script"}]},{"id":"XMfljB8QIHDMQh6951iY","title":"Scheduler","pathname":"/data-pipeline-2/components/scheduler","siteSpaceId":"sitesp_uyO3L","description":"A task can be scheduled to automatically execute at a given scheduler time.","breadcrumbs":[{"label":"Components"}]},{"id":"lYiNkuBuoNrcakIeBMUd","title":"Custom Components","pathname":"/data-pipeline-2/custom-components","siteSpaceId":"sitesp_uyO3L"},{"id":"s81bLic5j4MHBGmAWRWr","title":"Advance Configuration & Monitoring","pathname":"/data-pipeline-2/advance-configuration-and-monitoring","siteSpaceId":"sitesp_uyO3L","description":"This section explains the mandatory configurations for the Data Pipeline module to work. You can monitor critical things like the health of system components and Data Channels from this section."},{"id":"oXH9ILXDppbMfYu45yGd","title":"Configuration","pathname":"/data-pipeline-2/advance-configuration-and-monitoring/configuration","siteSpaceId":"sitesp_uyO3L","description":"This section provides Logger Settings, and Default Component Configuration options for the user to configure.","breadcrumbs":[{"label":"Advance Configuration & Monitoring"}]},{"id":"Q3RY6qDjVpDRiWlqKJaX","title":"Default Component Configuration","pathname":"/data-pipeline-2/advance-configuration-and-monitoring/configuration/default-component-configuration","siteSpaceId":"sitesp_uyO3L","breadcrumbs":[{"label":"Advance Configuration & Monitoring"},{"label":"Configuration"}]},{"id":"bxk9GITFhfvovzFvZXeY","title":"Logger","pathname":"/data-pipeline-2/advance-configuration-and-monitoring/configuration/logger","siteSpaceId":"sitesp_uyO3L","description":"This section provides information of the Kafka topic that is set for logging.","breadcrumbs":[{"label":"Advance Configuration & Monitoring"},{"label":"Configuration"}]},{"id":"WUENKE39rfCwQkugQDzI","title":"Data Channel","pathname":"/data-pipeline-2/advance-configuration-and-monitoring/data-channel","siteSpaceId":"sitesp_uyO3L","description":"The Data Channel page helps to check the state of the Kafka Brokers.","breadcrumbs":[{"label":"Advance Configuration & Monitoring"}]},{"id":"LYLM7QxtCDYsastv65Ed","title":"Cluster Events","pathname":"/data-pipeline-2/advance-configuration-and-monitoring/cluster-events","siteSpaceId":"sitesp_uyO3L","description":"This page enables a user to monitor all the events created in the pipeline environment.","breadcrumbs":[{"label":"Advance Configuration & Monitoring"}]},{"id":"gGE523maetXqrw7sM7cv","title":"System Component Status","pathname":"/data-pipeline-2/advance-configuration-and-monitoring/system-component-status","siteSpaceId":"sitesp_uyO3L","description":"This option allows to monitor the health of the system components.","breadcrumbs":[{"label":"Advance Configuration & Monitoring"}]},{"id":"6tJ1ZZDiBHGs39XxDpTt","title":"Version Control","pathname":"/data-pipeline-2/version-control","siteSpaceId":"sitesp_uyO3L","description":"The Version Control feature has been provided for the user to maintain a version of the pipeline while the same pipeline undergoes further development and different enhancements."},{"id":"QhP1MgEZTk9n8B9lX3pz","title":"Use Cases","pathname":"/data-pipeline-2/use-cases","siteSpaceId":"sitesp_uyO3L","description":"Refer to the below given YouTube videos to understand some actual scenarios on how the Data Pipeline functions."}]}