Pinot DB Schema

curl -H "Content-Type: application/json" -X POST -d "{\"sql\": \"select * from
wikipedia\"}" http://localhost:8099/query/sql
{
"schemaName": "wikipedia_user_bot_counts",
"dimensionFieldSpecs": [
{
"name": "user",
"dataType": "STRING"
},
{
"name": "bot",
"dataType": "STRING" // You can use "BOOLEAN" if "bot" represents Boolean
values
}
],
"metricFieldSpecs": [
{
"name": "record_count",
"dataType": "LONG"
}
]
}
{
"schemaName": "wikipedia_user_bot_counts",
"dimensionFieldSpecs": [
{"name": "user", "dataType": "STRING"},
{"name": "bot", "dataType": "STRING"} // Assuming you want "bot" as STRING
type
],
"metricFieldSpecs": [
{"name": "record_count", "dataType": "LONG"}
]
}
curl -H "Content-Type: application/json" -X POST

-d "{\"sql\": \"SELECT user, bot, COUNT(*) AS record_count FROM wikipedia GROUP BY
user, bot limit 100\" }" http://localhost:8099/query/sql > output.json
import subprocess
import json
import time
import psycopg2 # Assuming you're using PostgreSQL
import requests
# PostgreSQL connection parameters

db_params = {
"host": "localhost",
"port": 25432,
"database": "pino2",
"user": "user",
"password": "user",
}
conn = psycopg2.connect(**db_params)
# Function to run the Pinot query and save results to the database
def run_pinot_query_and_save():
try:
# Define the Pinot query in JSON format
pinot_query = {
"sql": "SELECT domain, count(user) AS changes FROM wikipedia GROUP BY
domain ORDER BY changes DESC LIMIT 10"
}
# Send the Pinot query to Pinot's SQL endpoint

response = requests.post(
"http://localhost:8099/query/sql",
headers={"Content-Type": "application/json"},
data=json.dumps(pinot_query),
)
# Parse the JSON response

query_result = response.json()
# Extract relevant data from the JSON response

domain = query_result['resultTable']['rows'][0][0] # Assuming you want the
first domain
changes = query_result['resultTable']['rows'][0][1] # Assuming you want
the first changes value
# Save the extracted data to the database

cursor = conn.cursor()
cursor.execute(
"INSERT INTO postg (domain, changes) VALUES (%s, %s)",
(domain, changes),
)
conn.commit()
cursor.close()
print("Query executed and result saved to the database.")

except Exception as e:
print(f"Error: {str(e)}")
# Run the query and save results every 10 seconds

while True:
run_pinot_query_and_save()
time.sleep(10) # Sleep for 10 seconds before the next query
docker exec -it kafka-wiki kafka-topics.sh --bootstrap-server localhost:9092 --

partitions 5 --topic wiki-events --create
docker exec -it monitoring-apache-pinot-master-pinot-controller-1 bin/pinot-
admin.sh AddTable -tableConfigFile /config/table.json -schemaFile
/config/schema.json -exec
curl -H "Content-Type: application/json" -X POST -d "{\"sql\": \"SELECT

oe.customer_id, oe.item,oe.location,oe.order_id,oe.quantity,oe.restaurant_id,
oe.restaurant_name, oe.ts as order_ts, ce.customer_name, ce.email, ce.phone, ce.ts
as customer_ts FROM order_events oe JOIN customer_events ce ON oe.customer_id =
ce.customer_id}" http://localhost:8099/query/sql > output.json
version: '3.7'
services:
zookeeper:
image: zookeeper:3.5.6
container_name: "zookeeper-wiki"
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
kafka:
image: wurstmeister/kafka:latest
restart: unless-stopped
container_name: "kafka-wiki"
ports:
- "9092:9092"
expose:
- "9093"
depends_on:
- zookeeper
environment:
KAFKA_ZOOKEEPER_CONNECT: zookeeper-wiki:2181/kafka
KAFKA_BROKER_ID: 0
KAFKA_ADVERTISED_HOST_NAME: kafka-wiki
KAFKA_ADVERTISED_LISTENERS:
PLAINTEXT://kafka-wiki:9093,OUTSIDE://localhost:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
pinot-controller:
image: apachepinot/pinot:0.12.0
command: "StartController -zkAddress zookeeper-wiki:2181 -dataDir /data"
container_name: "pinot-controller-wiki"
volumes:
- ./config:/config
- ./data:/data
ports:
- "9000:9000"
environment:
- PINOT_OPTS=-Dcom.sun.management.jmxremote -
Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.ssl=false -
Dcom.sun.management.jmxremote.authenticate=false
depends_on:
- zookeeper
pinot-broker:
command: "StartBroker -zkAddress zookeeper-wiki:2181"
container_name: "pinot-broker-wiki"
volumes:
- ./config:/config
ports:
- "8099:8099"
environment:
depends_on:
- pinot-controller
pinot-server:
command: "StartServer -zkAddress zookeeper-wiki:2181"
container_name: "pinot-server-wiki"
volumes:
- ./config:/config
ports:
- "8098:8098"
environment:
depends_on:
- pinot-broker

Pinot DB Schema

Uploaded by

Copyright:

Available Formats

Pinot DB Schema

Uploaded by

Document Information

Original Title

Copyright

Available Formats

Share this document

Share or Embed Document

Sharing Options

Did you find this document useful?

Is this content inappropriate?

Copyright:

Available Formats

Pinot DB Schema

Uploaded by

Copyright:

Available Formats

curl -H "Content-Type: application/json" -X POST -d "{\"sql\": \"select * from

curl -H "Content-Type: application/json" -X POST

# PostgreSQL connection parameters

# Send the Pinot query to Pinot's SQL endpoint

# Parse the JSON response

# Extract relevant data from the JSON response

# Save the extracted data to the database

print("Query executed and result saved to the database.")

# Run the query and save results every 10 seconds

docker exec -it kafka-wiki kafka-topics.sh --bootstrap-server localhost:9092 --

curl -H "Content-Type: application/json" -X POST -d "{\"sql\": \"SELECT

You might also like