diff --git a/README.md b/README.md index 5e5476f..4455784 100644 --- a/README.md +++ b/README.md @@ -1 +1,25 @@ -# CSTR_InfluxDB \ No newline at end of file +# CSTR_InfluxDB +This project aims to use Kafka, Faust, and InfluxDB to monitor a digital twin of a CSTR (continous stirred tank reactor) and a PID controller for a cooling jacket for the CSTR. + + + +`dockerized` contains a working example and integration with InfluxDB but the CSTR and controller logic is wrong. + +`dockerized2` contains the correct CSTR logic but there are issues with + +Both repos contain this structure: +├── docker-compose.yml +├── faust_app +│   ├── Dockerfile +│   ├── faust_app.py +│   └── requirements.txt +├── python-script +│   ├── Dockerfile +│   ├── cstr_controller.py +│   ├── kafka_consumer_test.py +│   ├── kafka_producer_test.py +│   └── requirements.txt +├── telegraf +│   └── telegraf.conf + +You can also run the CSTR logic sperately with `python3 pid_control.py` diff --git a/__pycache__/cstr_reactor.cpython-311.pyc b/__pycache__/cstr_reactor.cpython-311.pyc new file mode 100644 index 0000000..bae64a2 Binary files /dev/null and b/__pycache__/cstr_reactor.cpython-311.pyc differ diff --git a/cstr_reactor.py b/cstr_reactor.py new file mode 100644 index 0000000..d4a6f73 --- /dev/null +++ b/cstr_reactor.py @@ -0,0 +1,40 @@ +import numpy as np +from scipy.integrate import odeint + +# Define CSTR model +def cstr(x, t, u, Tf, Caf): + Ca = x[0] + T = x[1] + + q = 100 + V = 100 + rho = 1000 + Cp = 0.239 + mdelH = 5e4 + EoverR = 8750 + k0 = 7.2e10 + UA = 5e4 + rA = k0 * np.exp(-EoverR / T) * Ca + + dCadt = q / V * (Caf - Ca) - rA + dTdt = q / V * (Tf - T) \ + + mdelH / (rho * Cp) * rA \ + + UA / V / rho / Cp * (u - T) + + xdot = np.zeros(2) + xdot[0] = dCadt + xdot[1] = dTdt + return xdot + +# Simulation function +def simulate_cstr(u, Tf, Caf, x0, t): + Ca = np.ones(len(t)) * x0[0] + T = np.ones(len(t)) * x0[1] + for i in range(len(t) - 1): + ts = [t[i], t[i + 1]] + y = odeint(cstr, x0, ts, args=(u[i+1], Tf, Caf)) + Ca[i + 1] = y[-1][0] + T[i + 1] = y[-1][1] + x0[0] = Ca[i + 1] + x0[1] = T[i + 1] + return Ca, T diff --git a/data_doublet_steps.txt b/data_doublet_steps.txt new file mode 100644 index 0000000..8d01f29 --- /dev/null +++ b/data_doublet_steps.txt @@ -0,0 +1,301 @@ +0.000000000000000000e+00,3.000000000000000000e+02,3.244754434315989897e+02,8.772529460809700419e-01,2.500000000000000000e+02 +3.333333333333333287e-02,2.500000000000000000e+02,3.210539903874181960e+02,8.777833148509553984e-01,2.500000000000000000e+02 +6.666666666666666574e-02,2.500000000000000000e+02,3.177939378449736978e+02,8.791571628540204575e-01,2.500000000000000000e+02 +1.000000000000000056e-01,2.500000000000000000e+02,3.147264494833665935e+02,8.811140299384584429e-01,2.500000000000000000e+02 +1.333333333333333315e-01,2.500000000000000000e+02,3.118669011064122287e+02,8.834651032827367523e-01,2.500000000000000000e+02 +1.666666666666666574e-01,2.500000000000000000e+02,3.092195831846203760e+02,8.860752402522621418e-01,2.500000000000000000e+02 +2.000000000000000111e-01,2.500000000000000000e+02,3.067813956968976754e+02,8.888482819025830262e-01,2.500000000000000000e+02 +2.333333333333333370e-01,2.500000000000000000e+02,3.045445634549996043e+02,8.917159020675285985e-01,2.500000000000000000e+02 +2.666666666666666630e-01,2.500000000000000000e+02,3.024985463727602450e+02,8.946295130728433298e-01,2.565507355349831755e+02 +2.999999999999999889e-01,2.565507355349831755e+02,3.010676848483416848e+02,8.975432884848869586e-01,2.683249970902651285e+02 +3.333333333333333148e-01,2.683249970902651285e+02,3.005477413157566957e+02,9.004083162153169173e-01,2.723881440278407808e+02 +3.666666666666666408e-01,2.723881440278407808e+02,3.003448559572825047e+02,9.031947812622708183e-01,2.737968617462594239e+02 +4.000000000000000222e-01,2.737968617462594239e+02,3.002541194328377401e+02,9.058951862678322753e-01,2.742922371079579307e+02 +4.333333333333333481e-01,2.742922371079579307e+02,3.002047298841056318e+02,9.085088824266539520e-01,2.744723640105063396e+02 +4.666666666666666741e-01,2.744723640105063396e+02,3.001720062104486715e+02,9.110374307416426731e-01,2.745426980193443001e+02 +5.000000000000000000e-01,2.745426980193443001e+02,3.001471432618290578e+02,9.134831106454769367e-01,2.745739553869082670e+02 +5.333333333333333259e-01,2.745739553869082670e+02,3.001268292476118518e+02,9.158484022896785648e-01,2.745905874621826115e+02 +5.666666666666666519e-01,2.745905874621826115e+02,3.001096760334519331e+02,9.181358091844717961e-01,2.746011525151198498e+02 +5.999999999999999778e-01,2.746011525151198498e+02,3.000949894307559589e+02,9.203477982215125808e-01,2.746087362298697485e+02 +6.333333333333333037e-01,2.746087362298697485e+02,3.000823429337588095e+02,9.224867791828883501e-01,2.746145272535235904e+02 +6.666666666666666297e-01,2.746145272535235904e+02,3.000714275278852483e+02,9.245550979708848649e-01,3.486970087356147019e+02 +6.999999999999999556e-01,3.486970087356147019e+02,3.049982074212682051e+02,9.264196962467847740e-01,3.058984304926977984e+02 +7.333333333333332815e-01,3.058984304926977984e+02,3.066497517175093890e+02,9.280086337963689980e-01,2.911191777292556822e+02 +7.666666666666666075e-01,2.911191777292556822e+02,3.071779040821308513e+02,9.294576861227351605e-01,2.860020889154662882e+02 +8.000000000000000444e-01,2.860020889154662882e+02,3.073212655263133115e+02,9.308294087043491594e-01,2.842451504618106242e+02 +8.333333333333333703e-01,2.842451504618106242e+02,3.073358027416461482e+02,9.321480099941801267e-01,2.836582583217768274e+02 +8.666666666666666963e-01,2.836582583217768274e+02,3.073101824226170606e+02,9.334225765877059988e-01,2.834767404498618362e+02 +9.000000000000000222e-01,2.834767404498618362e+02,3.072747546941265568e+02,9.346567769533878911e-01,2.834335451444102887e+02 +9.333333333333333481e-01,2.834335451444102887e+02,3.072395566059128100e+02,9.358523840692066775e-01,2.834356745979270045e+02 +9.666666666666666741e-01,2.834356745979270045e+02,3.072075933356047699e+02,9.370105457271177274e-01,2.834510167749126026e+02 +1.000000000000000000e+00,2.834510167749126026e+02,3.071794792397713536e+02,9.381322146786460481e-01,2.834687098211531975e+02 +1.033333333333333215e+00,2.834687098211531975e+02,3.071550524906982673e+02,9.392182930519747952e-01,2.834852777796541545e+02 +1.066666666666666652e+00,2.834852777796541545e+02,3.071339319179366498e+02,9.402696769987658465e-01,2.834997840387031829e+02 +1.100000000000000089e+00,2.834997840387031829e+02,3.071157015069072145e+02,9.412872881575652384e-01,2.835121745270702718e+02 +1.133333333333333304e+00,2.835121745270702718e+02,3.070999812497177004e+02,9.422720170965321884e-01,2.835225748988248142e+02 +1.166666666666666741e+00,2.835225748988248142e+02,3.070864278512470378e+02,9.432247710810931496e-01,2.835312345258017785e+02 +1.199999999999999956e+00,2.835312345258017785e+02,3.070747426394135573e+02,9.441464551192421117e-01,2.835383932995443388e+02 +1.233333333333333393e+00,2.835383932995443388e+02,3.070646673927063262e+02,9.450379678313809428e-01,2.835442656369328915e+02 +1.266666666666666607e+00,2.835442656369328915e+02,3.070559793631132948e+02,9.459001983242094624e-01,2.835490386248749815e+02 +1.300000000000000044e+00,2.835490386248749815e+02,3.070484865737874429e+02,9.467340236127091968e-01,2.835528739664255227e+02 +1.333333333333333259e+00,2.835528739664255227e+02,3.070420236249950108e+02,9.475403065296890137e-01,3.500000000000000000e+02 +1.366666666666666696e+00,3.500000000000000000e+02,3.114853050057513997e+02,9.480941602101402577e-01,3.106125547653019225e+02 +1.399999999999999911e+00,3.106125547653019225e+02,3.129531578894349764e+02,9.482839419145562276e-01,2.984685973003643085e+02 +1.433333333333333348e+00,2.984685973003643085e+02,3.135008680915967147e+02,9.483268602435810379e-01,2.940835126973623233e+02 +1.466666666666666563e+00,2.940835126973623233e+02,3.137157509401920947e+02,9.483118567948064515e-01,2.924823075093501075e+02 +1.500000000000000000e+00,2.924823075093501075e+02,3.138081755703482827e+02,9.482740752071282531e-01,2.918873723576083421e+02 +1.533333333333333215e+00,2.918873723576083421e+02,3.138542483519824486e+02,9.482269767370323788e-01,2.916583745739603160e+02 +1.566666666666666652e+00,2.916583745739603160e+02,3.138817626391924591e+02,9.481758362347473268e-01,2.915636693023736257e+02 +1.600000000000000089e+00,2.915636693023736257e+02,3.139010181346706077e+02,9.481228433627689745e-01,2.915192752516065866e+02 +1.633333333333333304e+00,2.915192752516065866e+02,3.139159620863569558e+02,9.480690288916175934e-01,2.914945528792432583e+02 +1.666666666666666741e+00,2.914945528792432583e+02,3.139282110861106503e+02,9.480149618151880730e-01,2.914782062569964296e+02 +1.699999999999999956e+00,2.914782062569964296e+02,3.139385115940106061e+02,9.479610100179152177e-01,2.914659685924984274e+02 +1.733333333333333393e+00,2.914659685924984274e+02,3.139472722334029413e+02,9.479074398592506334e-01,2.914561420792255717e+02 +1.766666666666666607e+00,2.914561420792255717e+02,3.139547596740719655e+02,9.478544565027680058e-01,2.914479803382048431e+02 +1.800000000000000044e+00,2.914479803382048431e+02,3.139611723174823510e+02,9.478022221501805067e-01,2.914410999859131834e+02 +1.833333333333333259e+00,2.914410999859131834e+02,3.139666693636150967e+02,9.477508657551731996e-01,2.914352650551064130e+02 +1.866666666666666696e+00,2.914352650551064130e+02,3.139713833901458315e+02,9.477004892109324174e-01,2.914303067166232495e+02 +1.899999999999999911e+00,2.914303067166232495e+02,3.139754266722236480e+02,9.476511718815261442e-01,2.914260922855461331e+02 +1.933333333333333348e+00,2.914260922855461331e+02,3.139788949753785232e+02,9.476029742075469109e-01,2.914225124493763701e+02 +1.966666666666666563e+00,2.914225124493763701e+02,3.139818702281121432e+02,9.475559407210890628e-01,2.914194752393689782e+02 +2.000000000000000000e+00,2.914194752393689782e+02,3.139844226515168657e+02,9.475101025918826192e-01,3.500000000000000000e+02 +2.033333333333333215e+00,3.500000000000000000e+02,3.179403552980591598e+02,9.471137768319800587e-01,3.236309462351918569e+02 +2.066666666666666430e+00,3.236309462351918569e+02,3.198829204806716575e+02,9.461286003748039519e-01,3.071978815181909113e+02 +2.100000000000000089e+00,3.071978815181909113e+02,3.206108543223209040e+02,9.448550995019973087e-01,3.010002451153819152e+02 +2.133333333333333304e+00,3.010002451153819152e+02,3.208822694538531550e+02,9.434968915348137530e-01,2.986525269431896277e+02 +2.166666666666666519e+00,2.986525269431896277e+02,3.209806088436993718e+02,9.421380613071387300e-01,2.977705391872312930e+02 +2.200000000000000178e+00,2.977705391872312930e+02,3.210135274059908284e+02,9.408105337288923398e-01,2.974483147081322159e+02 +2.233333333333333393e+00,2.974483147081322159e+02,3.210220946236714212e+02,9.395253817776265670e-01,2.973394172416826109e+02 +2.266666666666666607e+00,2.973394172416826109e+02,3.210219589881775732e+02,9.382855521714706937e-01,2.973110700806179807e+02 +2.299999999999999822e+00,2.973110700806179807e+02,3.210190275002703970e+02,9.370909702794495422e-01,2.973124930430070663e+02 +2.333333333333333481e+00,2.973124930430070663e+02,3.210154853610293912e+02,9.359404602296449438e-01,2.973243297738808337e+02 +2.366666666666666696e+00,2.973243297738808337e+02,3.210121009684685873e+02,9.348324869617807842e-01,2.973392717802427114e+02 +2.399999999999999911e+00,2.973392717802427114e+02,3.210091093474406421e+02,9.337654352265458702e-01,2.973546192608038155e+02 +2.433333333333333126e+00,2.973546192608038155e+02,3.210065498101977255e+02,9.327377116484516506e-01,2.973694267531627133e+02 +2.466666666666666785e+00,2.973694267531627133e+02,3.210043942376040604e+02,9.317477795611339708e-01,2.973834094009395699e+02 +2.500000000000000000e+00,2.973834094009395699e+02,3.210025951624164122e+02,9.307941685976516277e-01,2.973965250036782209e+02 +2.533333333333333215e+00,2.973965250036782209e+02,3.210011032380872393e+02,9.298754750525144441e-01,2.974088152324369503e+02 +2.566666666666666430e+00,2.974088152324369503e+02,3.209998740862684485e+02,9.289903385059922192e-01,2.974203356988427913e+02 +2.600000000000000089e+00,2.974203356988427913e+02,3.209988662206635581e+02,9.281374988124705405e-01,2.974311712291594745e+02 +2.633333333333333304e+00,2.974311712291594745e+02,3.209980457524695225e+02,9.273157330009597077e-01,2.974413821702866016e+02 +2.666666666666666519e+00,2.974413821702866016e+02,3.209973835307421268e+02,9.265238709625195357e-01,3.500000000000000000e+02 +2.700000000000000178e+00,3.500000000000000000e+02,3.245889003676035713e+02,9.252226101916520840e-01,3.335215229076593459e+02 +2.733333333333333393e+00,3.335215229076593459e+02,3.269422780773131194e+02,9.229461857097432942e-01,3.132147524735844399e+02 +2.766666666666666607e+00,3.132147524735844399e+02,3.278507535533694295e+02,9.200925608526953070e-01,3.050265109845337861e+02 +2.799999999999999822e+00,3.050265109845337861e+02,3.281795490239597939e+02,9.170745384785367049e-01,3.017481884915334263e+02 +2.833333333333333481e+00,3.017481884915334263e+02,3.282746869480288865e+02,9.140767838403143219e-01,3.004993613231368386e+02 +2.866666666666666696e+00,3.004993613231368386e+02,3.282785933139364261e+02,9.111729243659041355e-01,3.000877566233223774e+02 +2.899999999999999911e+00,3.000877566233223774e+02,3.282500606412847333e+02,9.083878456929485035e-01,3.000141745935479776e+02 +2.933333333333333126e+00,3.000141745935479776e+02,3.282129364150350739e+02,9.057262655233532245e-01,3.000699739733767615e+02 +2.966666666666666785e+00,3.000699739733767615e+02,3.281764546408568890e+02,9.031850520809586724e-01,3.001690163047102828e+02 +3.000000000000000000e+00,3.001690163047102828e+02,3.281438024181599644e+02,9.007583641731070623e-01,3.002767079015313811e+02 +3.033333333333333215e+00,3.002767079015313811e+02,3.281157281696603150e+02,8.984397259673601033e-01,3.003799664723555338e+02 +3.066666666666666430e+00,3.003799664723555338e+02,3.280920504354634204e+02,8.962228207310914962e-01,3.004745414306546536e+02 +3.100000000000000089e+00,3.004745414306546536e+02,3.280722791660845132e+02,8.941017592462264796e-01,3.005596924592083496e+02 +3.133333333333333304e+00,3.005596924592083496e+02,3.280558638512369498e+02,8.920711391114195532e-01,3.006359796353119123e+02 +3.166666666666666519e+00,3.006359796353119123e+02,3.280422862863256341e+02,8.901260256609312416e-01,3.007043632215935531e+02 +3.200000000000000178e+00,3.007043632215935531e+02,3.280310898121179548e+02,8.882619075796102104e-01,3.007658506886825762e+02 +3.233333333333333393e+00,3.007658506886825762e+02,3.280218834869034481e+02,8.864746480458640665e-01,3.008213696027464721e+02 +3.266666666666666607e+00,3.008213696027464721e+02,3.280143371151569340e+02,8.847604389976905237e-01,3.008717316508062254e+02 +3.299999999999999822e+00,3.008717316508062254e+02,3.280081735901962929e+02,8.831157607956369171e-01,3.009176317920135943e+02 +3.333333333333333481e+00,3.009176317920135943e+02,3.280031610620005722e+02,8.815373475050004570e-01,3.500000000000000000e+02 +3.366666666666666696e+00,3.500000000000000000e+02,3.314164341811813870e+02,8.792016002512339501e-01,3.389164340082702438e+02 +3.399999999999999911e+00,3.389164340082702438e+02,3.340982272570665259e+02,8.752737123505133221e-01,3.153666626985373114e+02 +3.433333333333333126e+00,3.153666626985373114e+02,3.351988534376368420e+02,8.702724118236997430e-01,3.049347613764502398e+02 +3.466666666666666785e+00,3.049347613764502398e+02,3.355985929993649393e+02,8.649466926583799786e-01,3.004364457740580292e+02 +3.500000000000000000e+00,3.004364457740580292e+02,3.356857680724069155e+02,8.596722764016188068e-01,2.987070340707832656e+02 +3.533333333333333215e+00,2.987070340707832656e+02,3.356411982186470482e+02,8.546128244583309774e-01,2.982543137946679508e+02 +3.566666666666666430e+00,2.982543137946679508e+02,3.355495049910517764e+02,8.498254103000364434e-01,2.983603578438837758e+02 +3.600000000000000089e+00,2.983603578438837758e+02,3.354489908942844067e+02,8.453177682060497578e-01,2.986833475620112495e+02 +3.633333333333333304e+00,2.986833475620112495e+02,3.353556484232017851e+02,8.410770667219892305e-01,2.990659314389550332e+02 +3.666666666666666519e+00,2.990659314389550332e+02,3.352749794088414887e+02,8.370834161224885994e-01,2.994402189635671334e+02 +3.699999999999999734e+00,2.994402189635671334e+02,3.352077911490999327e+02,8.333158625844289125e-01,2.997805836179115317e+02 +3.733333333333333393e+00,2.997805836179115317e+02,3.351529805830510327e+02,8.297547909507204666e-01,3.000805271823412568e+02 +3.766666666666666607e+00,3.000805271823412568e+02,3.351088256512427961e+02,8.263826937966909369e-01,3.003415889052501484e+02 +3.799999999999999822e+00,3.003415889052501484e+02,3.350735507817543635e+02,8.231842417020807101e-01,3.005681955909104204e+02 +3.833333333333333481e+00,3.005681955909104204e+02,3.350455489683869814e+02,8.201460864767640135e-01,3.007653835746671689e+02 +3.866666666666666696e+00,3.007653835746671689e+02,3.350234476836715203e+02,8.172565884225223565e-01,3.009378739753653917e+02 +3.899999999999999911e+00,3.009378739753653917e+02,3.350061082142689770e+02,8.145055465795446592e-01,3.010897636620167077e+02 +3.933333333333333126e+00,3.010897636620167077e+02,3.349926000270266968e+02,8.118839602815686174e-01,3.012244813560512853e+02 +3.966666666666666785e+00,3.012244813560512853e+02,3.349821686699478960e+02,8.093838285329920668e-01,3.013448467041513936e+02 +4.000000000000000000e+00,3.013448467041513936e+02,3.349742055287450171e+02,8.069979638893615137e-01,3.500000000000000000e+02 +4.033333333333333215e+00,3.500000000000000000e+02,3.384421849610342292e+02,8.034627146743328341e-01,3.388309942330861873e+02 +4.066666666666666430e+00,3.388309942330861873e+02,3.413630763546668732e+02,7.974085743019114769e-01,3.127162383004456956e+02 +4.099999999999999645e+00,3.127162383004456956e+02,3.426727523589826774e+02,7.895398215113725859e-01,2.997150490368315445e+02 +4.133333333333332860e+00,2.997150490368315445e+02,3.431634295077626007e+02,7.810791551004711009e-01,2.936155865381352896e+02 +4.166666666666666963e+00,2.936155865381352896e+02,3.432334357393035589e+02,7.727370173113528029e-01,2.913064916994664486e+02 +4.200000000000000178e+00,2.913064916994664486e+02,3.431041055568330762e+02,7.648624087568133278e-01,2.910125224772657475e+02 +4.233333333333333393e+00,2.910125224772657475e+02,3.428996329572497643e+02,7.575836403502828054e-01,2.916880839688444667e+02 +4.266666666666666607e+00,2.916880839688444667e+02,3.426858013755958723e+02,7.509104076238583003e-01,2.927383071313344658e+02 +4.299999999999999822e+00,2.927383071313344658e+02,3.424934048233516251e+02,7.447980482354522591e-01,2.938499277774069469e+02 +4.333333333333333037e+00,2.938499277774069469e+02,3.423334675226706167e+02,7.391834205101951438e-01,2.948773879169897896e+02 +4.366666666666666252e+00,2.948773879169897896e+02,3.422068437489687653e+02,7.340026460089493776e-01,2.957678935514613272e+02 +4.400000000000000355e+00,2.957678935514613272e+02,3.421098702347791800e+02,7.291985464794550342e-01,2.965153063547078887e+02 +4.433333333333333570e+00,2.965153063547078887e+02,3.420374246694611884e+02,7.247227954632557978e-01,2.971338663369175492e+02 +4.466666666666666785e+00,2.971338663369175492e+02,3.419844151943342467e+02,7.205356545032839399e-01,2.976443958866543085e+02 +4.500000000000000000e+00,2.976443958866543085e+02,3.419463954689830985e+02,7.166047942532475235e-01,2.980677497431690881e+02 +4.533333333333333215e+00,2.980677497431690881e+02,3.419197326576280602e+02,7.129039184114281946e-01,2.984221671407071312e+02 +4.566666666666666430e+00,2.984221671407071312e+02,3.419015646755120201e+02,7.094115400685258033e-01,2.987226276847540021e+02 +4.599999999999999645e+00,2.987226276847540021e+02,3.418897086472196065e+02,7.061097864300595051e-01,2.989807811611326542e+02 +4.633333333333332860e+00,2.989807811611326542e+02,3.418825026509169902e+02,7.029836911548407175e-01,2.992057067452168440e+02 +4.666666666666666963e+00,2.992057067452168440e+02,3.418787008409116197e+02,7.000205044948518696e-01,3.500000000000000000e+02 +4.700000000000000178e+00,3.500000000000000000e+02,3.456157631897090141e+02,6.953232334126573377e-01,3.339345741438688719e+02 +4.733333333333333393e+00,3.339345741438688719e+02,3.487176840307589032e+02,6.868448292244635489e-01,3.056700866728804158e+02 +4.766666666666666607e+00,3.056700866728804158e+02,3.502554749541100705e+02,6.756027929021469758e-01,2.897768614160991092e+02 +4.799999999999999822e+00,2.897768614160991092e+02,3.508545653362842813e+02,6.634096857759442800e-01,2.817446255498072674e+02 +4.833333333333333037e+00,2.817446255498072674e+02,3.508837036684832924e+02,6.515072976554180384e-01,2.789363987701386236e+02 +4.866666666666666252e+00,2.789363987701386236e+02,3.506039261370375471e+02,6.405861251662818834e-01,2.793580174213392979e+02 +4.900000000000000355e+00,2.793580174213392979e+02,3.502009114396636278e+02,6.309057964702746713e-01,2.814609287270364462e+02 +4.933333333333333570e+00,2.814609287270364462e+02,3.497926344029294228e+02,6.224539455213448091e-01,2.841627742927956888e+02 +4.966666666666666785e+00,2.841627742927956888e+02,3.494406207265997750e+02,6.150852428583641895e-01,2.868195477223108583e+02 +5.000000000000000000e+00,2.868195477223108583e+02,3.491666908520958827e+02,6.086142503181106722e-01,2.891244923388603638e+02 +5.033333333333333215e+00,2.891244923388603638e+02,3.489695472053000458e+02,6.028646793698601014e-01,2.909860864464583301e+02 +5.066666666666666430e+00,2.909860864464583301e+02,3.488372676979856806e+02,5.976891353745938940e-01,2.924269920423411691e+02 +5.099999999999999645e+00,2.924269920423411691e+02,3.487549985676700430e+02,5.929727899650178990e-01,2.935169661849625413e+02 +5.133333333333332860e+00,2.935169661849625413e+02,3.487088617983393419e+02,5.886298373008220786e-01,2.943354622172976178e+02 +5.166666666666666963e+00,2.943354622172976178e+02,3.486874877054053172e+02,5.845976775114267721e-01,2.949540958239893484e+02 +5.200000000000000178e+00,2.949540958239893484e+02,3.486822700631358316e+02,5.808311058245546610e-01,2.954305640134836040e+02 +5.233333333333333393e+00,2.954305640134836040e+02,3.486870494203881776e+02,5.772974014981100410e-01,2.958082714352705693e+02 +5.266666666666666607e+00,2.958082714352705693e+02,3.486976119758538744e+02,5.739725213249631253e-01,2.961183354265646130e+02 +5.299999999999999822e+00,2.961183354265646130e+02,3.487111906538954145e+02,5.708383078683909595e-01,2.963822432641083537e+02 +5.333333333333333037e+00,2.963822432641083537e+02,3.487260427519537984e+02,5.678805242813781406e-01,3.500000000000000000e+02 +5.366666666666666252e+00,3.500000000000000000e+02,3.528137721379580398e+02,5.624716632439300534e-01,3.274336679969804891e+02 +5.400000000000000355e+00,3.274336679969804891e+02,3.560956861347639801e+02,5.518466403373358808e-01,2.969974841306113831e+02 +5.433333333333333570e+00,2.969974841306113831e+02,3.578656268679492314e+02,5.374679395850077146e-01,2.781379887731775398e+02 +5.466666666666666785e+00,2.781379887731775398e+02,3.585575179525097269e+02,5.218410642802467914e-01,2.683012267259578039e+02 +5.500000000000000000e+00,2.683012267259578039e+02,3.584841875127872299e+02,5.069432236601940645e-01,2.656298348291164757e+02 +5.533333333333333215e+00,2.656298348291164757e+02,3.579535914081761234e+02,4.939514507120928988e-01,2.678963331198325477e+02 +5.566666666666666430e+00,2.678963331198325477e+02,3.572478807740146181e+02,4.832515246796841302e-01,2.727312115534104464e+02 +5.599999999999999645e+00,2.727312115534104464e+02,3.565717013628605514e+02,4.746848769205614937e-01,2.782048392201738807e+02 +5.633333333333332860e+00,2.782048392201738807e+02,3.560326157331006698e+02,4.678452038519957945e-01,2.831391215070366343e+02 +5.666666666666666963e+00,2.831391215070366343e+02,3.556597154346584944e+02,4.622885801996424182e-01,2.870413990749404434e+02 +5.700000000000000178e+00,2.870413990749404434e+02,3.554364468462978266e+02,4.576355364029138784e-01,2.898628415612016624e+02 +5.733333333333333393e+00,2.898628415612016624e+02,3.553281870745975652e+02,4.536004147099608752e-01,2.917681513882408808e+02 +5.766666666666666607e+00,2.917681513882408808e+02,3.552987400306644759e+02,4.499855515632057434e-01,2.929853494854758083e+02 +5.799999999999999822e+00,2.929853494854758083e+02,3.553177460058416841e+02,4.466632831278392612e-01,2.937294835592126105e+02 +5.833333333333333037e+00,2.937294835592126105e+02,3.553626965299098970e+02,4.435565934510486419e-01,2.941734385513721008e+02 +5.866666666666666252e+00,2.941734385513721008e+02,3.554183859159776375e+02,4.406225218083552608e-01,2.944431575950217734e+02 +5.900000000000000355e+00,2.944431575950217734e+02,3.554754386292970025e+02,4.378393847244282377e-01,2.946233824165042279e+02 +5.933333333333333570e+00,2.946233824165042279e+02,3.555287122491732248e+02,4.351976306122145033e-01,2.947666953207265692e+02 +5.966666666666666785e+00,2.947666953207265692e+02,3.555759069426366068e+02,4.326937132940533171e-01,2.949025284856809321e+02 +6.000000000000000000e+00,2.949025284856809321e+02,3.556164801199158205e+02,4.303262876862731234e-01,3.500000000000000000e+02 +6.033333333333333215e+00,3.500000000000000000e+02,3.599524245282624406e+02,4.248679704422684278e-01,3.232373625037989768e+02 +6.066666666666666430e+00,3.232373625037989768e+02,3.634247865451787334e+02,4.129831694174113665e-01,2.905988528483258619e+02 +6.099999999999999645e+00,2.905988528483258619e+02,3.653682615110659526e+02,3.966514263071167501e-01,2.694593078371040633e+02 +6.133333333333332860e+00,2.694593078371040633e+02,3.660503767258605876e+02,3.792056210894841906e-01,2.590484712848373192e+02 +6.166666666666666963e+00,2.590484712848373192e+02,3.657533749109648511e+02,3.634195828749673396e-01,2.580797456361884770e+02 +6.200000000000000178e+00,2.580797456361884770e+02,3.648814089173848743e+02,3.508171135577168309e-01,2.635959611687407005e+02 +6.233333333333333393e+00,2.635959611687407005e+02,3.638560681360459057e+02,3.416188876199313107e-01,2.719106454389006444e+02 +6.266666666666666607e+00,2.719106454389006444e+02,3.629729454098790029e+02,3.352453768234771037e-01,2.801024097699998379e+02 +6.299999999999999822e+00,2.801024097699998379e+02,3.623573871238855872e+02,3.308748343924709512e-01,2.866530643804009628e+02 +6.333333333333333037e+00,2.866530643804009628e+02,3.620105253747290703e+02,3.277674666532369718e-01,2.911899642698025445e+02 +6.366666666666666252e+00,2.911899642698025445e+02,3.618751238912303734e+02,3.253735986604626707e-01,2.939566255367996632e+02 +6.400000000000000355e+00,2.939566255367996632e+02,3.618812918240581666e+02,3.233319015187455236e-01,2.954076340412798913e+02 +6.433333333333333570e+00,2.954076340412798913e+02,3.619682863891192142e+02,3.214307643937756587e-01,2.959949752759252419e+02 +6.466666666666666785e+00,2.959949752759252419e+02,3.620912236764483509e+02,3.195648568046808302e-01,2.960846856177141717e+02 +6.500000000000000000e+00,2.960846856177141717e+02,3.622206490631447195e+02,3.176979002206135605e-01,2.959400214312796606e+02 +6.533333333333333215e+00,2.959400214312796606e+02,3.623396220394062084e+02,3.158336456256446989e-01,2.957315111235712948e+02 +6.566666666666666430e+00,2.957315111235712948e+02,3.624402964243929546e+02,3.139948768230798049e-01,2.955562767889415454e+02 +6.599999999999999645e+00,2.955562767889415454e+02,3.625208222586069837e+02,3.122096803105590679e-01,2.954585614023587823e+02 +6.633333333333332860e+00,2.954585614023587823e+02,3.625828625385198620e+02,3.105032686033253886e-01,2.954479255099062129e+02 +6.666666666666666963e+00,2.954479255099062129e+02,3.626297368120507940e+02,3.088941857453573192e-01,3.500000000000000000e+02 +6.700000000000000178e+00,3.500000000000000000e+02,3.670002128561451400e+02,3.038516226617620775e-01,3.233412686190084173e+02 +6.733333333333333393e+00,3.233412686190084173e+02,3.706259340942795006e+02,2.917971840340646827e-01,2.890154084027560089e+02 +6.766666666666666607e+00,2.890154084027560089e+02,3.726002083449958491e+02,2.752749051349510645e-01,2.672787903132489760e+02 +6.799999999999999822e+00,2.672787903132489760e+02,3.730939302941353048e+02,2.585139197868814853e-01,2.585489843306595503e+02 +6.833333333333333037e+00,2.585489843306595503e+02,3.724679856421558952e+02,2.447275620698568244e-01,2.610026098110872681e+02 +6.866666666666666252e+00,2.610026098110872681e+02,3.713060172019652327e+02,2.351511436645561115e-01,2.699724869729990360e+02 +6.899999999999999467e+00,2.699724869729990360e+02,3.701504875354998489e+02,2.293654892322445071e-01,2.804405051979647396e+02 +6.933333333333333570e+00,2.804405051979647396e+02,3.692998708899031044e+02,2.262590723863167341e-01,2.892393618560371920e+02 +6.966666666666666785e+00,2.892393618560371920e+02,3.688181890073153113e+02,2.247312382622591864e-01,2.952805339519233030e+02 +7.000000000000000000e+00,2.952805339519233030e+02,3.686423621996511883e+02,2.239578069935770632e-01,2.987342594289866611e+02 +7.033333333333333215e+00,2.987342594289866611e+02,3.686719913816287431e+02,2.234128378298582684e-01,3.002507397268613545e+02 +7.066666666666666430e+00,3.002507397268613545e+02,3.688148046624363019e+02,2.228095908057473318e-01,3.005295030439546053e+02 +7.099999999999999645e+00,3.005295030439546053e+02,3.690017720965783496e+02,2.220301685448618223e-01,3.001484964040633940e+02 +7.133333333333332860e+00,3.001484964040633940e+02,3.691881871970199427e+02,2.210633990028244011e-01,2.995213114027403094e+02 +7.166666666666666963e+00,2.995213114027403094e+02,3.693495741620283184e+02,2.199558660008663624e-01,2.989077093508774396e+02 +7.200000000000000178e+00,2.989077093508774396e+02,3.694763511181104150e+02,2.187761138987106835e-01,2.984428278294897154e+02 +7.233333333333333393e+00,2.984428278294897154e+02,3.695687004515664285e+02,2.175912020052666040e-01,2.981713894176572239e+02 +7.266666666666666607e+00,2.981713894176572239e+02,3.696321778083527647e+02,2.164539937838293460e-01,2.980810079704711484e+02 +7.299999999999999822e+00,2.980810079704711484e+02,3.696743658714428307e+02,2.153988560060644819e-01,2.981303573354732066e+02 +7.333333333333333037e+00,2.981303573354732066e+02,3.697026658926024538e+02,2.144426907342061317e-01,3.500000000000000000e+02 +7.366666666666666252e+00,3.500000000000000000e+02,3.738658222789007937e+02,2.100941307344187658e-01,3.282908954203256826e+02 +7.399999999999999467e+00,3.282908954203256826e+02,3.775955067637755747e+02,1.988940071858600056e-01,2.930459895633558176e+02 +7.433333333333333570e+00,2.930459895633558176e+02,3.794636804211351659e+02,1.838798258051424495e-01,2.724732026568510719e+02 +7.466666666666666785e+00,2.724732026568510719e+02,3.796639248057171585e+02,1.699361005759597865e-01,2.670331757487350615e+02 +7.500000000000000000e+00,2.670331757487350615e+02,3.787710039159784401e+02,1.599685904895331301e-01,2.728917224679142919e+02 +7.533333333333333215e+00,2.728917224679142919e+02,3.775481216162842770e+02,1.543171399702262525e-01,2.834457237653742254e+02 +7.566666666666666430e+00,2.834457237653742254e+02,3.765355645793611075e+02,1.518705489909124573e-01,2.934223329417154673e+02 +7.599999999999999645e+00,2.934223329417154673e+02,3.759269714048319315e+02,1.512737202364933253e-01,3.004888479660247640e+02 +7.633333333333332860e+00,3.004888479660247640e+02,3.756905157421375065e+02,1.514715426231309225e-01,3.044375486615775799e+02 +7.666666666666666963e+00,3.044375486615775799e+02,3.757115318305076244e+02,1.518007991222248554e-01,3.059802688400591251e+02 +7.700000000000000178e+00,3.059802688400591251e+02,3.758726280412467986e+02,1.519229535290840261e-01,3.060122531424020735e+02 +7.733333333333333393e+00,3.060122531424020735e+02,3.760836104863558944e+02,1.517268899810959581e-01,3.052991628156423189e+02 +7.766666666666666607e+00,3.052991628156423189e+02,3.762867610466273618e+02,1.512396149984186478e-01,3.043845593327357619e+02 +7.799999999999999822e+00,3.043845593327357619e+02,3.764525908260824849e+02,1.505540924871982267e-01,3.035910679048292877e+02 +7.833333333333333037e+00,3.035910679048292877e+02,3.765724596753208857e+02,1.497773634707225132e-01,3.030604301738972595e+02 +7.866666666666666252e+00,3.030604301738972595e+02,3.766507376500925375e+02,1.489994113079959881e-01,3.028083529479649201e+02 +7.899999999999999467e+00,3.028083529479649201e+02,3.766979352941157799e+02,1.482799190912053866e-01,3.027796702816310699e+02 +7.933333333333333570e+00,3.027796702816310699e+02,3.767255999170242831e+02,1.476482817240883649e-01,3.028940768579447536e+02 +7.966666666666666785e+00,3.028940768579447536e+02,3.767433637742131509e+02,1.471104425959039730e-01,3.030759683078160265e+02 +8.000000000000000000e+00,3.030759683078160265e+02,3.767577772970931846e+02,1.466578966299347253e-01,3.500000000000000000e+02 +8.033333333333333215e+00,3.500000000000000000e+02,3.805085713482093865e+02,1.431482283244117082e-01,3.376543068756357115e+02 +8.066666666666666430e+00,3.376543068756357115e+02,3.843439381210681063e+02,1.334679298263795610e-01,3.017725689777702769e+02 +8.099999999999999645e+00,3.017725689777702769e+02,3.860644357305905601e+02,1.209123604572908545e-01,2.831016765301721989e+02 +8.133333333333332860e+00,2.831016765301721989e+02,3.860122403495737444e+02,1.105626680176429455e-01,2.808712735474130682e+02 +8.166666666666666075e+00,2.808712735474130682e+02,3.850324676907488310e+02,1.044332360638335899e-01,2.885273754169940048e+02 +8.199999999999999289e+00,2.885273754169940048e+02,3.839671894200151883e+02,1.019198655123021574e-01,2.984090343897167372e+02 +8.233333333333332504e+00,2.984090343897167372e+02,3.832436716708909898e+02,1.015618911078990938e-01,3.061099359672805917e+02 +8.266666666666665719e+00,3.061099359672805917e+02,3.829235303085007445e+02,1.021008667030627315e-01,3.105173487771968439e+02 +8.300000000000000711e+00,3.105173487771968439e+02,3.829023387134833456e+02,1.027462736653871056e-01,3.121926416788811594e+02 +8.333333333333333925e+00,3.121926416788811594e+02,3.830449812715667690e+02,1.031206266052455689e-01,3.121627141117861015e+02 +8.366666666666667140e+00,3.121627141117861015e+02,3.832427349576629467e+02,1.031310879033096872e-01,3.113572938048755532e+02 +8.400000000000000355e+00,3.113572938048755532e+02,3.834276655689911308e+02,1.028441684120450089e-01,3.104210116957592618e+02 +8.433333333333333570e+00,3.104210116957592618e+02,3.835688232334059649e+02,1.023859313795245640e-01,3.096986817660841780e+02 +8.466666666666666785e+00,3.096986817660841780e+02,3.836615633617587378e+02,1.018764571272674224e-01,3.092984595992281811e+02 +8.500000000000000000e+00,3.092984595992281811e+02,3.837156493749141077e+02,1.013984570431603477e-01,3.091822877514591141e+02 +8.533333333333333215e+00,3.091822877514591141e+02,3.837453771280324304e+02,1.009922235236523913e-01,3.092509833527848286e+02 +8.566666666666666430e+00,3.092509833527848286e+02,3.837633968588531275e+02,1.006653667585042516e-01,3.094035082356564885e+02 +8.599999999999999645e+00,3.094035082356564885e+02,3.837781277216236617e+02,1.004068526142278123e-01,3.095665480169661805e+02 +8.633333333333332860e+00,3.095665480169661805e+02,3.837936972876735808e+02,1.001990710587707251e-01,3.097008556424305539e+02 +8.666666666666666075e+00,3.097008556424305539e+02,3.838111203132520473e+02,1.000255101959232479e-01,3.500000000000000000e+02 +8.699999999999999289e+00,3.500000000000000000e+02,3.870208616330907603e+02,9.729669446206869188e-02,3.499052250082547744e+02 +8.733333333333332504e+00,3.499052250082547744e+02,3.909917996229627306e+02,8.930609800807659537e-02,3.132462059197237068e+02 +8.766666666666665719e+00,3.132462059197237068e+02,3.925796596528014106e+02,7.938796277451951044e-02,2.964536258692041315e+02 +8.800000000000000711e+00,2.964536258692041315e+02,3.923734634262583540e+02,7.238045608715201751e-02,2.965064027910050868e+02 +8.833333333333333925e+00,2.965064027910050868e+02,3.914669675961212647e+02,6.918837737164658386e-02,3.042480895764705338e+02 +8.866666666666667140e+00,3.042480895764705338e+02,3.906685928512919190e+02,6.855223851490808984e-02,3.120674923540863119e+02 +8.900000000000000355e+00,3.120674923540863119e+02,3.902476122955503683e+02,6.902702392339228610e-02,3.169692684632252053e+02 +8.933333333333333570e+00,3.169692684632252053e+02,3.901554921916022636e+02,6.967384849091368126e-02,3.189583194451967074e+02 +8.966666666666666785e+00,3.189583194451967074e+02,3.902512980480385636e+02,7.006764314201165134e-02,3.190828051699033949e+02 +9.000000000000000000e+00,3.190828051699033949e+02,3.904106797750593500e+02,7.012818551087114660e-02,3.184053531681790901e+02 +9.033333333333333215e+00,3.184053531681790901e+02,3.905580908466751566e+02,6.994591695647606189e-02,3.176397422092486522e+02 +9.066666666666666430e+00,3.176397422092486522e+02,3.906637883373269347e+02,6.965286575329528163e-02,3.171168646607401342e+02 +9.099999999999999645e+00,3.171168646607401342e+02,3.907278846150917389e+02,6.935216086093180798e-02,3.168917593613611530e+02 +9.133333333333332860e+00,3.168917593613611530e+02,3.907634814516166557e+02,6.909729915750889573e-02,3.168818524604947697e+02 +9.166666666666666075e+00,3.168818524604947697e+02,3.907847605467443373e+02,6.890114125189972238e-02,3.169754811126725826e+02 +9.199999999999999289e+00,3.169754811126725826e+02,3.908015194662595491e+02,6.875424259471511756e-02,3.170882615843920007e+02 +9.233333333333332504e+00,3.170882615843920007e+02,3.908183971041240170e+02,6.864053206795921180e-02,3.171771954615450113e+02 +9.266666666666665719e+00,3.171771954615450113e+02,3.908363517178968323e+02,6.854625975082186129e-02,3.172319818932836597e+02 +9.300000000000000711e+00,3.172319818932836597e+02,3.908545952333635114e+02,6.846293030237274313e-02,3.172595090132996347e+02 +9.333333333333333925e+00,3.172595090132996347e+02,3.908719864819038321e+02,6.838674211385896429e-02,3.500000000000000000e+02 +9.366666666666667140e+00,3.500000000000000000e+02,3.934552079705911751e+02,6.636553917717626094e-02,3.500000000000000000e+02 +9.400000000000000355e+00,3.500000000000000000e+02,3.964855069019390044e+02,6.094254585882852293e-02,3.319439757387670511e+02 +9.433333333333333570e+00,3.319439757387670511e+02,3.982654579523800749e+02,5.447511838781046029e-02,3.151489928043273494e+02 +9.466666666666666785e+00,3.151489928043273494e+02,3.983969948142385533e+02,4.999372792885729139e-02,3.133991702002182365e+02 +9.500000000000000000e+00,3.133991702002182365e+02,3.978993544653387744e+02,4.813852968939997395e-02,3.181303514230850169e+02 +9.533333333333333215e+00,3.181303514230850169e+02,3.974596932081474279e+02,4.785648762909828352e-02,3.229187612888035801e+02 +9.566666666666666430e+00,3.229187612888035801e+02,3.972753983707168004e+02,4.810771342090628538e-02,3.255973822050352737e+02 +9.599999999999999645e+00,3.255973822050352737e+02,3.972909962241147923e+02,4.833103264780790054e-02,3.264090464328755843e+02 +9.633333333333332860e+00,3.264090464328755843e+02,3.973967643388844522e+02,4.835961261069689210e-02,3.262454527200055736e+02 +9.666666666666666075e+00,3.262454527200055736e+02,3.975129982314302310e+02,4.823010626646760668e-02,3.258285336860301413e+02 +9.699999999999999289e+00,3.258285336860301413e+02,3.976045650742314592e+02,4.803336658612402527e-02,3.255159774164645228e+02 +9.733333333333332504e+00,3.255159774164645228e+02,3.976672719949200427e+02,4.783874398394660776e-02,3.253854053046750323e+02 +9.766666666666665719e+00,3.253854053046750323e+02,3.977098803229791315e+02,4.767667929908785823e-02,3.253830018445770520e+02 +9.800000000000000711e+00,3.253830018445770520e+02,3.977420954039355365e+02,4.754994564728081546e-02,3.254331515502186676e+02 +9.833333333333333925e+00,3.254331515502186676e+02,3.977699611681088641e+02,4.744978423076740004e-02,3.254859037465333245e+02 +9.866666666666667140e+00,3.254859037465333245e+02,3.977958268886618498e+02,4.736664509836860382e-02,3.255222597731305427e+02 +9.900000000000000355e+00,3.255222597731305427e+02,3.978199512173791277e+02,4.729425831858118795e-02,3.255421783968557747e+02 +9.933333333333333570e+00,3.255421783968557747e+02,3.978419404534390083e+02,4.722965136003608178e-02,3.255521732984906293e+02 +9.966666666666666785e+00,3.255521732984906293e+02,3.978615011147641667e+02,4.717180551135722461e-02,3.255582282778576655e+02 +1.000000000000000000e+01,3.255582282778576655e+02,3.978786335664927947e+02,4.712038368810446409e-02,3.255582282778576655e+02 diff --git a/dockerized/docker-compose.yml b/dockerized/docker-compose.yml new file mode 100644 index 0000000..848741c --- /dev/null +++ b/dockerized/docker-compose.yml @@ -0,0 +1,93 @@ +services: + zookeeper: + image: confluentinc/cp-zookeeper:latest + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + networks: + - kafka-net + + kafka: + image: confluentinc/cp-kafka:latest + depends_on: + - zookeeper + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + networks: + - kafka-net + healthcheck: + test: ["CMD", "nc", "-z", "localhost", "9092"] + interval: 10s + timeout: 10s + retries: 5 + + kafka-connect: + image: confluentinc/cp-kafka-connect:latest + depends_on: + - kafka + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: kafka:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "quickstart-avro" + CONNECT_CONFIG_STORAGE_TOPIC: "quickstart-avro-config" + CONNECT_OFFSET_STORAGE_TOPIC: "quickstart-avro-offsets" + CONNECT_STATUS_STORAGE_TOPIC: "quickstart-avro-status" + CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" + CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" + CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect" + CONNECT_PLUGIN_PATH: "/usr/share/java,/etc/kafka-connect/jars" + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + networks: + - kafka-net + + telegraf: + image: telegraf:latest + volumes: + - ./telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + kafka: + condition: service_healthy + networks: + - kafka-net + + python-script: + build: + context: ./python-script + dockerfile: Dockerfile + depends_on: + kafka: + condition: service_healthy + environment: + KAFKA_BROKER_ADDRESS: kafka:9092 + networks: + - kafka-net + healthcheck: + test: ["CMD-SHELL", "test -f /healthcheck"] + interval: 10s + timeout: 5s + retries: 5 + + faust-app: + build: + context: ./faust_app + dockerfile: Dockerfile + depends_on: + python-script: + condition: service_healthy + environment: + KAFKA_BROKER_ADDRESS: kafka:9092 + networks: + - kafka-net + +networks: + kafka-net: + driver: bridge diff --git a/dockerized/faust_app/Dockerfile b/dockerized/faust_app/Dockerfile new file mode 100644 index 0000000..05a2846 --- /dev/null +++ b/dockerized/faust_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.9-slim + +WORKDIR /app + +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["faust", "-A", "faust_app", "worker", "-l", "info"] diff --git a/dockerized/faust_app/faust_app.py b/dockerized/faust_app/faust_app.py new file mode 100644 index 0000000..089459c --- /dev/null +++ b/dockerized/faust_app/faust_app.py @@ -0,0 +1,47 @@ +import faust +import json +import logging + +class CSTRData(faust.Record, serializer='json'): + Ca: float + Reactor_Temperature: float + +app = faust.App('cstr-controller', broker='kafka://kafka:9092') +data_topic = app.topic('cstr_data', value_type=CSTRData) +tc_topic = app.topic('cstr_control') + +# Maintain integral of error (ie) globally +ie = 0 + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +@app.agent(data_topic) +async def process(stream): + global ie + async for event in stream: + ca = event.Ca + temp_reactor = event.Reactor_Temperature + + # Compute Tc value based on your control logic + tc = compute_tc(ca, temp_reactor) + + # Produce the result to the control topic + await tc_topic.send(value=json.dumps({"Tc": tc})) + +def compute_tc(ca, temp_reactor): + global ie + Kc = 4.61730615181 * 2.0 + tauI = 0.913444964569 / 4.0 + e = temp_reactor - 324.475443431599 + ie += e # Update integral of error + tc = 300.0 + Kc * e + Kc / tauI * ie + + # Log the PID components and Tc value + logger.info(f"e: {e}, ie: {ie}, P: {Kc * e}, I: {Kc / tauI * ie}, tc: {tc}") + + return tc + +if __name__ == '__main__': + app.main() diff --git a/dockerized/faust_app/requirements.txt b/dockerized/faust_app/requirements.txt new file mode 100644 index 0000000..73f1221 --- /dev/null +++ b/dockerized/faust_app/requirements.txt @@ -0,0 +1,2 @@ +faust +kafka-python diff --git a/dockerized/python-script/Dockerfile b/dockerized/python-script/Dockerfile new file mode 100644 index 0000000..607672a --- /dev/null +++ b/dockerized/python-script/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.9-slim + +WORKDIR /app + +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Debugging: List the contents of the /app directory +RUN ls -al /app + +# CMD ["python", "kafka_producer_test.py"] # Change this to test the producer +# CMD ["python", "kafka_consumer_test.py"] # Change this to test the consumer +CMD ["python", "cstr_controller.py"] + +# Keep the container running for debugging +# CMD ["tail", "-f", "/dev/null"] diff --git a/dockerized/python-script/cstr_controller.py b/dockerized/python-script/cstr_controller.py new file mode 100644 index 0000000..2524d46 --- /dev/null +++ b/dockerized/python-script/cstr_controller.py @@ -0,0 +1,217 @@ +import logging +import numpy as np +from scipy.integrate import odeint +from kafka import KafkaProducer, KafkaConsumer +import json +import os + +# Configure logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +# Kafka setup +producer = KafkaProducer( + bootstrap_servers=os.getenv('KAFKA_BROKER_ADDRESS'), + value_serializer=lambda v: json.dumps(v).encode('utf-8') +) + +consumer = KafkaConsumer( + 'cstr_control', + bootstrap_servers=os.getenv('KAFKA_BROKER_ADDRESS'), + auto_offset_reset='earliest', + enable_auto_commit=False, + value_deserializer=lambda m: json.loads(m.decode('utf-8')) if m else None, + consumer_timeout_ms=10000, # Increased timeout +) + +# Send a test message to Kafka +test_data = {"Tc": 300.0} +producer.send('cstr_control', value=test_data) +producer.flush() +logger.info(f"Sent test data to Kafka: {test_data}") + +# Initial Tc value +initial_tc = 300.0 + +# from IMC tuning +Kc = 4.61730615181 * 2.0 +tauI = 0.913444964569 / 4.0 +tauD = 0.0 + +# define CSTR model +def cstr(x, t, u, Tf, Caf): + Tc = u + Ca = x[0] + T = x[1] + + q = 100 + V = 100 + rho = 1000 + Cp = 0.239 + mdelH = 5e4 + EoverR = 8750 + k0 = 7.2e10 + UA = 5e4 + rA = k0 * np.exp(-EoverR / T) * Ca + + dCadt = q / V * (Caf - Ca) - rA + dTdt = q / V * (Tf - T) + mdelH / (rho * Cp) * rA + UA / V / rho / Cp * (Tc - T) + + xdot = np.zeros(2) + xdot[0] = dCadt + xdot[1] = dTdt + return xdot + +Ca_ss = 0.87725294608097 +T_ss = 324.475443431599 +x0 = np.empty(2) +x0[0] = Ca_ss +x0[1] = T_ss + +u_ss = initial_tc +Tf = 350 +Caf = 1 + +t = np.linspace(0, 10, 301) + +Ca = np.ones(len(t)) * Ca_ss +T = np.ones(len(t)) * T_ss +u = np.ones(len(t)) * u_ss + +op = np.ones(len(t)) * u_ss +pv = np.zeros(len(t)) +e = np.zeros(len(t)) +ie = np.zeros(len(t)) +dpv = np.zeros(len(t)) +P = np.zeros(len(t)) +I = np.zeros(len(t)) +D = np.zeros(len(t)) +sp = np.ones(len(t)) * T_ss + +for i in range(15): + sp[i * 20:(i + 1) * 20] = 300 + i * 7.0 +sp[300] = sp[299] + +op_hi = 350.0 +op_lo = 250.0 + +pv[0] = T_ss + +# Function to send data to Kafka +def send_data_to_kafka(ca, temp_reactor): + if not np.isnan(ca) and not np.isnan(temp_reactor): + data = { + "Ca": ca, + "Reactor_Temperature": temp_reactor + } + producer.send('cstr_data', value=data) + producer.flush() + logger.info(f"Sent data to Kafka: {data}") + else: + logger.error(f"Attempted to send NaN values to Kafka: Ca={ca}, Reactor_Temperature={temp_reactor}") + +# Function to receive Tc from Kafka +def receive_tc_from_kafka(): + logger.info("Waiting to receive message from Kafka...") + for attempt in range(5): # Retry up to 5 times + for message in consumer: + logger.debug(f"Raw message from Kafka: {message}") + if message.value is not None: + logger.info(f"Received message from Kafka: {message.value}") + try: + value = message.value + if isinstance(value, str): + value = json.loads(value) + if not np.isnan(value["Tc"]): + return value["Tc"] + else: + logger.error(f"Received NaN value for Tc: {value}") + except (KeyError, json.JSONDecodeError) as e: + logger.error(f"Error processing message: {e}") + else: + logger.warning("Received an empty message or invalid JSON") + logger.info(f"Attempt {attempt + 1} failed, retrying...") + consumer.poll(timeout_ms=5000) + logger.info("Exiting receive_tc_from_kafka after 5 attempts") + return None + +logger.info("Starting CSTR simulation loop") + +# Initial iteration with predefined Tc +logger.info("Initial iteration with predefined Tc") +ts = [0, t[1]] +y = odeint(cstr, x0, ts, args=(initial_tc, Tf, Caf)) +Ca[1] = y[-1][0] +T[1] = y[-1][1] +x0[0] = Ca[1] +x0[1] = T[1] +pv[1] = T[1] + +send_data_to_kafka(Ca[1], T[1]) + +# Ready to start-up faust to send msgs and faust can go collect +# the initial value. +if not os.path.isfile("/healthcheck"): + with open("/healthcheck", "w") as f: + f.write("healthcheck") + +# Main simulation loop +for i in range(1, len(t) - 1): + logger.info(f"Iteration {i} starting") + delta_t = t[i + 1] - t[i] + e[i] = sp[i] - pv[i] + if i >= 1: + dpv[i] = (pv[i] - pv[i - 1]) / delta_t + ie[i] = ie[i - 1] + e[i] * delta_t + P[i] = Kc * e[i] + I[i] = Kc / tauI * ie[i] + D[i] = -Kc * tauD * dpv[i] + op[i] = op[0] + P[i] + I[i] + D[i] + if op[i] > op_hi: + op[i] = op_hi + ie[i] = ie[i] - e[i] * delta_t + if op[i] < op_lo: + op[i] = op_lo + ie[i] = ie[i] - e[i] * delta_t + ts = [t[i], t[i + 1]] + logger.info("Waiting to receive message from Kafka...") + try: + u[i + 1] = receive_tc_from_kafka() + if u[i + 1] is None: + logger.error("No valid Tc value received, breaking loop.") + break + except Exception as e: + logger.error(f"Error receiving Tc: {e}") + break + y = odeint(cstr, x0, ts, args=(u[i + 1], Tf, Caf)) + Ca[i + 1] = y[-1][0] + T[i + 1] = y[-1][1] + if not np.isnan(Ca[i + 1]) and not np.isnan(T[i + 1]): + x0[0] = Ca[i + 1] + x0[1] = T[i + 1] + pv[i + 1] = T[i + 1] + send_data_to_kafka(Ca[i + 1], T[i + 1]) + else: + logger.error(f"Encountered NaN values in iteration {i}: Ca={Ca[i + 1]}, T={T[i + 1]}") + break + +op[len(t) - 1] = op[len(t) - 2] +ie[len(t) - 1] = ie[len(t) - 2] +P[len(t) - 1] = P[len(t) - 2] +I[len(t) - 1] = I[len(t) - 2] +D[len(t) - 1] = D[len(t) - 2] + +data = np.vstack((t, u, T)).T +np.savetxt('data_doublet_steps.txt', data, delimiter=',') + +producer.close() +consumer.close() + +# Add a simple test run to check message reception +if __name__ == "__main__": + logger.info("Starting test run...") + try: + tc = receive_tc_from_kafka() + logger.info(f"Received Tc: {tc}") + except Exception as e: + logger.error(f"Error receiving message from Kafka: {e}") diff --git a/dockerized/python-script/kafka_consumer_test.py b/dockerized/python-script/kafka_consumer_test.py new file mode 100644 index 0000000..b51a1fb --- /dev/null +++ b/dockerized/python-script/kafka_consumer_test.py @@ -0,0 +1,35 @@ +# kafka_consumer_test.py +import logging +import json +from kafka import KafkaConsumer +import os + +# Configure logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +# Kafka setup +consumer = KafkaConsumer( + 'cstr_control', + bootstrap_servers=os.getenv('KAFKA_BROKER_ADDRESS'), + auto_offset_reset='earliest', + enable_auto_commit=False, + value_deserializer=lambda m: json.loads(m.decode('utf-8')) if m else None, + consumer_timeout_ms=10000, # Increased timeout +) + +# Function to receive a test message from Kafka +def receive_test_message_from_kafka(): + logger.info("Waiting to receive test message from Kafka...") + for message in consumer: + logger.debug(f"Raw message from Kafka: {message}") + if message.value is not None: + logger.info(f"Received test message from Kafka: {message.value}") + return message.value + else: + logger.warning("Received an empty message or invalid JSON") + +# Receive the test message +test_message = receive_test_message_from_kafka() +logger.info(f"Test message received: {test_message}") + diff --git a/dockerized/python-script/kafka_producer_test.py b/dockerized/python-script/kafka_producer_test.py new file mode 100644 index 0000000..9137d48 --- /dev/null +++ b/dockerized/python-script/kafka_producer_test.py @@ -0,0 +1,21 @@ +# kafka_producer_test.py +import logging +import json +from kafka import KafkaProducer +import os + +# Configure logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +# Kafka setup +producer = KafkaProducer( + bootstrap_servers=os.getenv('KAFKA_BROKER_ADDRESS'), + value_serializer=lambda v: json.dumps(v).encode('utf-8') +) + +# Send a test message to Kafka +test_data = {"Tc": 300.0} +producer.send('cstr_control', value=test_data) +producer.flush() +logger.info(f"Sent test data to Kafka: {test_data}") diff --git a/dockerized/python-script/requirements.txt b/dockerized/python-script/requirements.txt new file mode 100644 index 0000000..487c357 --- /dev/null +++ b/dockerized/python-script/requirements.txt @@ -0,0 +1,5 @@ +numpy +matplotlib +scipy +kafka-python +faust diff --git a/dockerized/telegraf/telegraf.conf b/dockerized/telegraf/telegraf.conf new file mode 100644 index 0000000..f037013 --- /dev/null +++ b/dockerized/telegraf/telegraf.conf @@ -0,0 +1,29 @@ +[agent] + interval = "10s" + round_interval = true + debug = true + +[[inputs.kafka_consumer]] + brokers = ["kafka:9092"] + topics = ["cstr_data"] + data_format = "json_v2" + + [[inputs.kafka_consumer.json_v2]] + measurement_name = "cstr_data" + [[inputs.kafka_consumer.json_v2.field]] + path = "Ca" + type = "float" + [[inputs.kafka_consumer.json_v2.field]] + path = "Reactor_Temperature" + type = "float" + +[[outputs.file]] + files = ["stdout"] + + +[[outputs.influxdb_v2]] + urls = ["https://us-east-1-1.aws.cloud2.influxdata.com/"] + token = "d-17JiPCEX66F0t7F3NNnfdZjPAF4tp6DmFyC8VFmizmBOi874Ao_bdwW_7wklicSqMqZCcNXzaOffU5bXFj9Q==" + organization = "89711f17730122e0" + bucket = "CSTR" + diff --git a/dockerized2/docker-compose.yml b/dockerized2/docker-compose.yml new file mode 100644 index 0000000..848741c --- /dev/null +++ b/dockerized2/docker-compose.yml @@ -0,0 +1,93 @@ +services: + zookeeper: + image: confluentinc/cp-zookeeper:latest + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + networks: + - kafka-net + + kafka: + image: confluentinc/cp-kafka:latest + depends_on: + - zookeeper + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + networks: + - kafka-net + healthcheck: + test: ["CMD", "nc", "-z", "localhost", "9092"] + interval: 10s + timeout: 10s + retries: 5 + + kafka-connect: + image: confluentinc/cp-kafka-connect:latest + depends_on: + - kafka + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: kafka:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "quickstart-avro" + CONNECT_CONFIG_STORAGE_TOPIC: "quickstart-avro-config" + CONNECT_OFFSET_STORAGE_TOPIC: "quickstart-avro-offsets" + CONNECT_STATUS_STORAGE_TOPIC: "quickstart-avro-status" + CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" + CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" + CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect" + CONNECT_PLUGIN_PATH: "/usr/share/java,/etc/kafka-connect/jars" + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + networks: + - kafka-net + + telegraf: + image: telegraf:latest + volumes: + - ./telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + kafka: + condition: service_healthy + networks: + - kafka-net + + python-script: + build: + context: ./python-script + dockerfile: Dockerfile + depends_on: + kafka: + condition: service_healthy + environment: + KAFKA_BROKER_ADDRESS: kafka:9092 + networks: + - kafka-net + healthcheck: + test: ["CMD-SHELL", "test -f /healthcheck"] + interval: 10s + timeout: 5s + retries: 5 + + faust-app: + build: + context: ./faust_app + dockerfile: Dockerfile + depends_on: + python-script: + condition: service_healthy + environment: + KAFKA_BROKER_ADDRESS: kafka:9092 + networks: + - kafka-net + +networks: + kafka-net: + driver: bridge diff --git a/dockerized2/faust_app/Dockerfile b/dockerized2/faust_app/Dockerfile new file mode 100644 index 0000000..05a2846 --- /dev/null +++ b/dockerized2/faust_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.9-slim + +WORKDIR /app + +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["faust", "-A", "faust_app", "worker", "-l", "info"] diff --git a/dockerized2/faust_app/faust_app.py b/dockerized2/faust_app/faust_app.py new file mode 100644 index 0000000..cbd6fbd --- /dev/null +++ b/dockerized2/faust_app/faust_app.py @@ -0,0 +1,200 @@ +import faust +import logging +import json +import numpy as np +from scipy.integrate import odeint + +class CSTRData(faust.Record, serializer='json'): + Ca: float + Reactor_Temperature: float + +app = faust.App('cstr-controller', broker='kafka://kafka:9092') +data_topic = app.topic('cstr_data', value_type=CSTRData) +tc_topic = app.topic('cstr_control') + +# PID parameters +Kc = 4.61730615181 * 2.0 +tauI = 0.913444964569 / 4.0 +tauD = 0.0 + +# Control loop function +def pid_control(T_ss, u_ss, t, Tf, Caf, x0): + u = np.ones(len(t)) * u_ss + op = np.ones(len(t)) * u_ss + pv = np.zeros(len(t)) + e = np.zeros(len(t)) + ie = np.zeros(len(t)) + dpv = np.zeros(len(t)) + P = np.zeros(len(t)) + I = np.zeros(len(t)) + D = np.zeros(len(t)) + + # Initialize Ca and T arrays + Ca = np.ones(len(t)) * x0[0] + T = np.ones(len(t)) * x0[1] + + # Upper and Lower limits on OP + op_hi = 350.0 + op_lo = 250.0 + + pv[0] = T_ss + + # Define the setpoint ramp or steps + sp = np.ones(len(t)) * T_ss + for i in range(15): + sp[i * 20:(i + 1) * 20] = 300 + i * 7.0 + sp[300] = sp[299] + + for i in range(len(t) - 1): + delta_t = t[i + 1] - t[i] + e[i] = sp[i] - pv[i] + if i >= 1: + dpv[i] = (pv[i] - pv[i - 1]) / delta_t + ie[i] = ie[i - 1] + e[i] * delta_t + P[i] = Kc * e[i] + I[i] = Kc / tauI * ie[i] + D[i] = -Kc * tauD * dpv[i] + op[i] = op[0] + P[i] + I[i] + D[i] + if op[i] > op_hi: + op[i] = op_hi + ie[i] = ie[i] - e[i] * delta_t + if op[i] < op_lo: + op[i] = op_lo + ie[i] = ie[i] - e[i] * delta_t + u[i + 1] = op[i] + + # Use the current Ca and T for the initial condition of the next step + x0 = [Ca[i], T[i]] + ts = [t[i], t[i + 1]] + y = odeint(cstr, x0, ts, args=(u[i + 1], Tf, Caf)) # Use u[i + 1] + Ca[i + 1] = y[-1][0] + T[i + 1] = y[-1][1] + pv[i + 1] = T[i + 1] + + # Send Tc to Kafka + send_tc_to_kafka(op[i]) + + # Debugging information + if i % 50 == 0: + print(f"Time: {t[i]:.2f}, Setpoint: {sp[i]:.2f}, PV: {pv[i]:.2f}, OP: {op[i]:.2f}, Ca: {Ca[i]:.2f}, T: {T[i]:.2f}") + + op[len(t) - 1] = op[len(t) - 2] + ie[len(t) - 1] = ie[len(t) - 2] + P[len(t) - 1] = P[len(t) - 2] + I[len(t) - 1] = I[len(t) - 2] + D[len(t) - 1] = D[len(t) - 2] + + return u, T + +# Define CSTR model +def cstr(x, t, u, Tf, Caf): + Ca = x[0] + T = x[1] + + q = 100 + V = 100 + rho = 1000 + Cp = 0.239 + mdelH = 5e4 + EoverR = 8750 + k0 = 7.2e10 + UA = 5e4 + rA = k0 * np.exp(-EoverR / T) * Ca + + dCadt = q / V (Caf - Ca) - rA + dTdt = q / V (Tf - T) + mdelH / (rho * Cp) * rA + UA / V / rho / Cp * (u - T) + + xdot = np.zeros(2) + xdot[0] = dCadt + xdot[1] = dTdt + return xdot + +# Maintain integral of error (ie) globally +ie = 0 + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Function to send Tc value to Kafka +def send_tc_to_kafka(tc_value): + producer = KafkaProducer(bootstrap_servers='kafka:9092', value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('cstr_control', value={"Tc": tc_value}) + producer.flush() + +@app.agent(data_topic) +async def process(stream): + async for event in stream: + ca = event.Ca + temp_reactor = event.Reactor_Temperature + + # Define the PID control parameters and initial conditions + T_ss = 300 # Steady-state temperature + u_ss = 300 # Steady-state control input + t = np.linspace(0, 20, 300) # Time vector + Tf = 300 # Feed temperature + Caf = 1 # Feed concentration + x0 = [ca, temp_reactor] # Initial conditions + + # Compute Tc value based on your control logic + _, T = pid_control(T_ss, u_ss, t, Tf, Caf, x0) + tc = T[-1] # Use the last temperature value as the control signal + + # Produce the result to the control topic + await tc_topic.send(value={"Tc": tc}) + +if __name__ == '__main__': + app.main() + + +# Function to send Tc to Kafka +def send_tc_to_kafka(tc): + if not np.isnan(tc): + data = { + "Tc": tc + } + producer.send('cstr_control', value=data) + producer.flush() + logger.info(f"Sent Tc to Kafka: {data}") + else: + logger.error(f"Attempted to send NaN value to Kafka: Tc={tc}") + +# Function to receive data from Kafka +def receive_data_from_kafka(): + logger.info("Waiting to receive message from Kafka...") + for attempt in range(5): # Retry up to 5 times + for message in consumer: + logger.debug(f"Raw message from Kafka: {message}") + if message.value is not None: + logger.info(f"Received message from Kafka: {message.value}") + try: + value = message.value + if isinstance(value, str): + value = json.loads(value) + if not np.isnan(value["Ca"]) and not np.isnan(value["Reactor_Temperature"]): + return value["Ca"], value["Reactor_Temperature"] + else: + logger.error(f"Received NaN value for Ca or Reactor_Temperature: {value}") + except (KeyError, json.JSONDecodeError) as e: + logger.error(f"Error processing message: {e}") + else: + logger.warning("Received an empty message or invalid JSON") + logger.info(f"Attempt {attempt + 1} failed, retrying...") + consumer.poll(timeout_ms=5000) + logger.info("Exiting receive_data_from_kafka after 5 attempts") + return None, None + +# Main execution +if __name__ == "__main__": + t = np.linspace(0, 10, 301) + u_ss = 300.0 + T_ss = 324.475443431599 + + while True: + Ca, T = receive_data_from_kafka() + if Ca is not None and T is not None: + x0 = [Ca, T] + u, T = pid_control(T_ss, u_ss, t, 350, 1, x0) + else: + logger.error("No valid Ca and T values received, terminating loop") + break diff --git a/dockerized2/faust_app/requirements.txt b/dockerized2/faust_app/requirements.txt new file mode 100644 index 0000000..73f1221 --- /dev/null +++ b/dockerized2/faust_app/requirements.txt @@ -0,0 +1,2 @@ +faust +kafka-python diff --git a/dockerized2/python-script/Dockerfile b/dockerized2/python-script/Dockerfile new file mode 100644 index 0000000..607672a --- /dev/null +++ b/dockerized2/python-script/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.9-slim + +WORKDIR /app + +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Debugging: List the contents of the /app directory +RUN ls -al /app + +# CMD ["python", "kafka_producer_test.py"] # Change this to test the producer +# CMD ["python", "kafka_consumer_test.py"] # Change this to test the consumer +CMD ["python", "cstr_controller.py"] + +# Keep the container running for debugging +# CMD ["tail", "-f", "/dev/null"] diff --git a/dockerized2/python-script/cstr_controller.py b/dockerized2/python-script/cstr_controller.py new file mode 100644 index 0000000..b54eafd --- /dev/null +++ b/dockerized2/python-script/cstr_controller.py @@ -0,0 +1,130 @@ +import numpy as np +from scipy.integrate import odeint +from kafka import KafkaProducer, KafkaConsumer +import json +import logging +import os + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Kafka setup +producer = KafkaProducer( + bootstrap_servers=os.getenv('KAFKA_BROKER_ADDRESS'), + value_serializer=lambda v: json.dumps(v).encode('utf-8') +) + +consumer = KafkaConsumer( + 'cstr_control', + bootstrap_servers=os.getenv('KAFKA_BROKER_ADDRESS'), + auto_offset_reset='earliest', + enable_auto_commit=False, + value_deserializer=lambda m: json.loads(m.decode('utf-8')) if m else None, + consumer_timeout_ms=10000, # Increased timeout +) + +# Define CSTR model +def cstr(x, t, u, Tf, Caf): + Ca = x[0] + T = x[1] + + q = 100 + V = 100 + rho = 1000 + Cp = 0.239 + mdelH = 5e4 + EoverR = 8750 + k0 = 7.2e10 + UA = 5e4 + rA = k0 * np.exp(-EoverR / T) * Ca + + dCadt = q / V * (Caf - Ca) - rA + dTdt = q / V * (Tf - T) + mdelH / (rho * Cp) * rA + UA / V / rho / Cp * (u - T) + + xdot = np.zeros(2) + xdot[0] = dCadt + xdot[1] = dTdt + return xdot + +# Function to send data to Kafka +def send_data_to_kafka(ca, temp_reactor): + if not np.isnan(ca) and not np.isnan(temp_reactor): + data = { + "Ca": float(ca), + "Reactor_Temperature": float(temp_reactor) + } + producer.send('cstr_data', value=data) + producer.flush() + logger.info(f"Sent data to Kafka: {data}") + else: + logger.error(f"Attempted to send NaN values to Kafka: Ca={ca}, Reactor_Temperature={temp_reactor}") + +# Function to receive Tc from Kafka +def receive_tc_from_kafka(): + logger.info("Waiting to receive message from Kafka...") + for attempt in range(5): # Retry up to 5 times + for message in consumer: + logger.debug(f"Raw message from Kafka: {message}") + if message.value is not None: + logger.info(f"Received message from Kafka: {message.value}") + try: + value = message.value + if isinstance(value, str): + value = json.loads(value) + if not np.isnan(value["Tc"]): + return value["Tc"] + else: + logger.error(f"Received NaN value for Tc: {value}") + except (KeyError, json.JSONDecodeError) as e: + logger.error(f"Error processing message: {e}") + else: + logger.warning("Received an empty message or invalid JSON") + logger.info(f"Attempt {attempt + 1} failed, retrying...") + consumer.poll(timeout_ms=5000) + logger.info("Exiting receive_tc_from_kafka after 5 attempts") + return None + +# Simulation function +def simulate_cstr(u, Tf, Caf, x0, t): + Ca = np.ones(len(t)) * x0[0] + T = np.ones(len(t)) * x0[1] + for i in range(len(t) - 1): + ts = [t[i], t[i + 1]] + y = odeint(cstr, x0, ts, args=(u[i+1], Tf, Caf)) + Ca[i + 1] = y[-1][0] + T[i + 1] = y[-1][1] + x0[0] = Ca[i + 1] + x0[1] = T[i + 1] + + logger.debug(f"Iteration {i}: Ca={Ca[i + 1]}, T={T[i + 1]}") + + # Send data to Kafka + send_data_to_kafka(Ca[i + 1], T[i + 1]) + + # Receive Tc from Kafka + tc = receive_tc_from_kafka() + if tc is not None: + u[i + 1] = tc + else: + logger.error("No valid Tc value received, using previous value") + u[i + 1] = u[i] + + return Ca, T + +# Main loop to continuously run the simulation +if __name__ == "__main__": + t = np.linspace(0, 10, 301) + x0 = [0.87725294608097, 324.475443431599] + u_ss = 300.0 + + while True: + # Initial Tc value + initial_tc = 300.0 + u = np.ones(301) * initial_tc + + # Run simulation + Ca, T = simulate_cstr(u, 350, 1, x0, t) + + # Update x0 for the next iteration + x0 = [Ca[-1], T[-1]] diff --git a/dockerized2/python-script/kafka_consumer_test.py b/dockerized2/python-script/kafka_consumer_test.py new file mode 100644 index 0000000..b51a1fb --- /dev/null +++ b/dockerized2/python-script/kafka_consumer_test.py @@ -0,0 +1,35 @@ +# kafka_consumer_test.py +import logging +import json +from kafka import KafkaConsumer +import os + +# Configure logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +# Kafka setup +consumer = KafkaConsumer( + 'cstr_control', + bootstrap_servers=os.getenv('KAFKA_BROKER_ADDRESS'), + auto_offset_reset='earliest', + enable_auto_commit=False, + value_deserializer=lambda m: json.loads(m.decode('utf-8')) if m else None, + consumer_timeout_ms=10000, # Increased timeout +) + +# Function to receive a test message from Kafka +def receive_test_message_from_kafka(): + logger.info("Waiting to receive test message from Kafka...") + for message in consumer: + logger.debug(f"Raw message from Kafka: {message}") + if message.value is not None: + logger.info(f"Received test message from Kafka: {message.value}") + return message.value + else: + logger.warning("Received an empty message or invalid JSON") + +# Receive the test message +test_message = receive_test_message_from_kafka() +logger.info(f"Test message received: {test_message}") + diff --git a/dockerized2/python-script/kafka_producer_test.py b/dockerized2/python-script/kafka_producer_test.py new file mode 100644 index 0000000..9137d48 --- /dev/null +++ b/dockerized2/python-script/kafka_producer_test.py @@ -0,0 +1,21 @@ +# kafka_producer_test.py +import logging +import json +from kafka import KafkaProducer +import os + +# Configure logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +# Kafka setup +producer = KafkaProducer( + bootstrap_servers=os.getenv('KAFKA_BROKER_ADDRESS'), + value_serializer=lambda v: json.dumps(v).encode('utf-8') +) + +# Send a test message to Kafka +test_data = {"Tc": 300.0} +producer.send('cstr_control', value=test_data) +producer.flush() +logger.info(f"Sent test data to Kafka: {test_data}") diff --git a/dockerized2/python-script/requirements.txt b/dockerized2/python-script/requirements.txt new file mode 100644 index 0000000..487c357 --- /dev/null +++ b/dockerized2/python-script/requirements.txt @@ -0,0 +1,5 @@ +numpy +matplotlib +scipy +kafka-python +faust diff --git a/dockerized2/telegraf/telegraf.conf b/dockerized2/telegraf/telegraf.conf new file mode 100644 index 0000000..f037013 --- /dev/null +++ b/dockerized2/telegraf/telegraf.conf @@ -0,0 +1,29 @@ +[agent] + interval = "10s" + round_interval = true + debug = true + +[[inputs.kafka_consumer]] + brokers = ["kafka:9092"] + topics = ["cstr_data"] + data_format = "json_v2" + + [[inputs.kafka_consumer.json_v2]] + measurement_name = "cstr_data" + [[inputs.kafka_consumer.json_v2.field]] + path = "Ca" + type = "float" + [[inputs.kafka_consumer.json_v2.field]] + path = "Reactor_Temperature" + type = "float" + +[[outputs.file]] + files = ["stdout"] + + +[[outputs.influxdb_v2]] + urls = ["https://us-east-1-1.aws.cloud2.influxdata.com/"] + token = "d-17JiPCEX66F0t7F3NNnfdZjPAF4tp6DmFyC8VFmizmBOi874Ao_bdwW_7wklicSqMqZCcNXzaOffU5bXFj9Q==" + organization = "89711f17730122e0" + bucket = "CSTR" + diff --git a/pid_control.py b/pid_control.py new file mode 100644 index 0000000..fa04f61 --- /dev/null +++ b/pid_control.py @@ -0,0 +1,113 @@ +import numpy as np +import matplotlib.pyplot as plt +from cstr_reactor import simulate_cstr, cstr # Ensure cstr is imported +from scipy.integrate import odeint + + +# PID parameters and initial conditions +Kc = 4.61730615181 * 2.0 +tauI = 0.913444964569 / 4.0 +tauD = 0.0 + +# Control loop +def pid_control(T_ss, u_ss, t, Tf, Caf, x0): + u = np.ones(len(t)) * u_ss + op = np.ones(len(t)) * u_ss + pv = np.zeros(len(t)) + e = np.zeros(len(t)) + ie = np.zeros(len(t)) + dpv = np.zeros(len(t)) + P = np.zeros(len(t)) + I = np.zeros(len(t)) + D = np.zeros(len(t)) + + # Initialize Ca and T arrays + Ca = np.ones(len(t)) * x0[0] + T = np.ones(len(t)) * x0[1] + + # Upper and Lower limits on OP + op_hi = 350.0 + op_lo = 250.0 + + pv[0] = T_ss + + # Define the setpoint ramp or steps + sp = np.ones(len(t)) * T_ss + for i in range(15): + sp[i * 20:(i + 1) * 20] = 300 + i * 7.0 + sp[300] = sp[299] + + # Create plot + plt.figure(figsize=(10, 7)) + plt.ion() + plt.show() + + for i in range(len(t) - 1): + delta_t = t[i + 1] - t[i] + e[i] = sp[i] - pv[i] + if i >= 1: + dpv[i] = (pv[i] - pv[i - 1]) / delta_t + ie[i] = ie[i - 1] + e[i] * delta_t + P[i] = Kc * e[i] + I[i] = Kc / tauI * ie[i] + D[i] = -Kc * tauD * dpv[i] + op[i] = op[0] + P[i] + I[i] + D[i] + if op[i] > op_hi: + op[i] = op_hi + ie[i] = ie[i] - e[i] * delta_t + if op[i] < op_lo: + op[i] = op_lo + ie[i] = ie[i] - e[i] * delta_t + u[i + 1] = op[i] + + # Use the current Ca and T for the initial condition of the next step + x0 = [Ca[i], T[i]] + ts = [t[i], t[i+1]] + y = odeint(cstr, x0, ts, args=(u[i + 1], Tf, Caf)) # Use u[i + 1] + Ca[i + 1] = y[-1][0] + T[i + 1] = y[-1][1] + pv[i + 1] = T[i + 1] + + # Debugging information + if i % 50 == 0: + print(f"Time: {t[i]:.2f}, Setpoint: {sp[i]:.2f}, PV: {pv[i]:.2f}, OP: {op[i]:.2f}, Ca: {Ca[i]:.2f}, T: {T[i]:.2f}") + + # Plotting + plt.clf() + plt.subplot(2, 1, 1) + plt.plot(t[:i + 1], sp[:i + 1], 'r--', label='Setpoint') + plt.plot(t[:i + 1], pv[:i + 1], 'b-', label='Process Variable') + plt.ylabel('Temperature CSTR') + plt.legend(loc='best') + + plt.subplot(2, 1, 2) + plt.plot(t[:i + 1], op[:i + 1], 'k-', label='Control Output') + plt.ylabel('Temperature Reactor') + plt.xlabel('Time (sec)') + plt.legend(loc='best') + + plt.pause(0.01) + + op[len(t) - 1] = op[len(t) - 2] + ie[len(t) - 1] = ie[len(t) - 2] + P[len(t) - 1] = P[len(t) - 2] + I[len(t) - 1] = I[len(t) - 2] + D[len(t) - 1] = D[len(t) - 2] + + # Save data to file + data = np.vstack((t, u, T, Ca, op)).T + np.savetxt('data_doublet_steps.txt', data, delimiter=',') + + plt.ioff() + plt.show() + + return u, T + +# Main execution +if __name__ == "__main__": + t = np.linspace(0, 10, 301) + x0 = [0.87725294608097, 324.475443431599] + u_ss = 300.0 + T_ss = 324.475443431599 + + u, T = pid_control(T_ss, u_ss, t, 350, 1, x0)