{ "type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [ { "id": "WQNi7orzkmZF94NxStNjF", "type": "rectangle", "x": -11410.372758162084, "y": -5931.580697179193, "width": 87.76312163726138, "height": 60.004214960299514, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3M", "roundness": { "type": 3 }, "seed": 777051346, "version": 140, "versionNonce": 2033073426, "isDeleted": false, "boundElements": [ { "type": "text", "id": "Si3tdUJIZQ_JzME48fWpu" }, { "id": "zn35auLWSG8ZSvr_VDq8h", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false }, { "id": "Si3tdUJIZQ_JzME48fWpu", "type": "text", "x": -11398.051263566598, "y": -5921.578589699043, "width": 63.12013244628906, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3N", "roundness": null, "seed": 1434546834, "version": 85, "versionNonce": 1447611090, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false, "text": "sm\nlim: 3GB", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "WQNi7orzkmZF94NxStNjF", "originalText": "sm\nlim: 3GB", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "ID8KraUblzU5CxCAPUlHI", "type": "rectangle", "x": -11410.466676510621, "y": -5832.146995676874, "width": 91.12015099183466, "height": 63.30454286791064, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3O", "roundness": { "type": 3 }, "seed": 2107199570, "version": 135, "versionNonce": 1414242450, "isDeleted": false, "boundElements": [ { "type": "text", "id": "W3UygikdV9zT-r0CY4ncC" }, { "id": "IozgRfCgusmWAj1VxqBv2", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false }, { "id": "W3UygikdV9zT-r0CY4ncC", "type": "text", "x": -11396.72266394195, "y": -5820.494724242919, "width": 63.63212585449219, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3P", "roundness": null, "seed": 443512338, "version": 66, "versionNonce": 1055768146, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false, "text": "md\nlim: 6GB", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "ID8KraUblzU5CxCAPUlHI", "originalText": "md\nlim: 6GB", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "mjUQRLG1mG8sMIUYCuO31", "type": "rectangle", "x": -11403.527763650694, "y": -5731.35670477482, "width": 93.03852874903463, "height": 61.38621779892139, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3Q", "roundness": { "type": 3 }, "seed": 217768914, "version": 174, "versionNonce": 1392911378, "isDeleted": false, "boundElements": [ { "type": "text", "id": "x_QmNPZFuzgE3f-KpjDY_" }, { "id": "o0FV2LZFgtX1l-v7d378d", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false }, { "id": "x_QmNPZFuzgE3f-KpjDY_", "type": "text", "x": -11395.920578133599, "y": -5720.6635958753595, "width": 77.82415771484375, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3R", "roundness": null, "seed": 2136119698, "version": 88, "versionNonce": 445370834, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false, "text": "lg\nlim: 12 GB", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "mjUQRLG1mG8sMIUYCuO31", "originalText": "lg\nlim: 12 GB", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "oJ2a_LxQRXJru3pENR3Ch", "type": "text", "x": -11402.031108275021, "y": -5653.550814384389, "width": 281.24853515625, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3S", "roundness": null, "seed": 1085118290, "version": 209, "versionNonce": 1217775506, "isDeleted": false, "boundElements": [ { "id": "hj2NK9DE6SNZ19DKslH17", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false, "text": "Containers with varying Memory limit\nexecuting varying CPU bound jobs ", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Containers with varying Memory limit\nexecuting varying CPU bound jobs ", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Myv5MZEGhzImarZ4DewhH", "type": "rectangle", "x": -12321.541714224457, "y": -5852.291374425532, "width": 58.19703760049899, "height": 49.797471958635015, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3T", "roundness": { "type": 3 }, "seed": 560127250, "version": 324, "versionNonce": 358286674, "isDeleted": false, "boundElements": [ { "type": "text", "id": "Eycqa_fEg8B95OZ8Fwti9" }, { "id": "roLE-vulHyb90HkGBeWOg", "type": "arrow" }, { "id": "pCi_eGr_nvYIvpW9aV1wa", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false }, { "id": "Eycqa_fEg8B95OZ8Fwti9", "type": "text", "x": -12306.23522575868, "y": -5837.392638446215, "width": 27.584060668945312, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3U", "roundness": null, "seed": 1323920082, "version": 210, "versionNonce": 439520018, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896905, "link": null, "locked": false, "text": "Job", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "Myv5MZEGhzImarZ4DewhH", "originalText": "Job", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "WuAGAhUfojP0VfA-n-jHV", "type": "line", "x": -12293.343177134959, "y": -5800.09404187083, "width": 5.684341886080802e-14, "height": 212.98921191166943, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3V", "roundness": { "type": 2 }, "seed": 802701458, "version": 150, "versionNonce": 764631250, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -5.684341886080802e-14, 212.98921191166943 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "vjPQyk2ooQcpCduRDjefL", "type": "rectangle", "x": -12429.53628969821, "y": -5585.904887302136, "width": 274.7860445259339, "height": 119.39394832221433, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3W", "roundness": { "type": 3 }, "seed": 1072062034, "version": 211, "versionNonce": 1058303634, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false }, { "id": "sDwDmfZy7dS_TlIizUzsk", "type": "text", "x": -12399.597837403888, "y": -5564.70647709142, "width": 218.0644073486328, "height": 80, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3X", "roundness": null, "seed": 1810547730, "version": 182, "versionNonce": 862768210, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "text": "Characteristics:\n- CPU bound\n- Non-Preemptive by default\n- Long time to complete", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Characteristics:\n- CPU bound\n- Non-Preemptive by default\n- Long time to complete", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "xRfLT-cGKH79iSUz6kUFQ", "type": "rectangle", "x": -12181.748872562059, "y": -5866.090673784681, "width": 228.58840053837304, "height": 82.79579615488746, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3Y", "roundness": { "type": 3 }, "seed": 1843163602, "version": 391, "versionNonce": 1081307666, "isDeleted": false, "boundElements": [ { "id": "roLE-vulHyb90HkGBeWOg", "type": "arrow" }, { "id": "tHbixK6S0pl77GX5HKpv1", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false }, { "id": "dALiusMFhGC6C0jeU0SDg", "type": "rectangle", "x": -12171.849370359587, "y": -5853.191314939647, "width": 58.19703760049899, "height": 49.797471958635015, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3Z", "roundness": { "type": 3 }, "seed": 684877714, "version": 311, "versionNonce": 1226792914, "isDeleted": false, "boundElements": [ { "type": "text", "id": "hY_D0oMYFB_6mmkWB-MFl" } ], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false }, { "id": "hY_D0oMYFB_6mmkWB-MFl", "type": "text", "x": -12156.54288189381, "y": -5838.29257896033, "width": 27.584060668945312, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3a", "roundness": null, "seed": 84125010, "version": 201, "versionNonce": 965186962, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "text": "Job", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "dALiusMFhGC6C0jeU0SDg", "originalText": "Job", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "yJudd5ELiC-hhAXu63fkP", "type": "rectangle", "x": -12103.452902567651, "y": -5854.391274075326, "width": 58.19703760049899, "height": 49.797471958635015, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3b", "roundness": { "type": 3 }, "seed": 1271343890, "version": 309, "versionNonce": 1659048786, "isDeleted": false, "boundElements": [ { "type": "text", "id": "fx61kra-19UvYOK2JAkrt" } ], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false }, { "id": "fx61kra-19UvYOK2JAkrt", "type": "text", "x": -12088.146414101875, "y": -5839.4925380960085, "width": 27.584060668945312, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3c", "roundness": null, "seed": 60029138, "version": 200, "versionNonce": 1468586258, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "text": "Job", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "yJudd5ELiC-hhAXu63fkP", "originalText": "Job", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "spNX_P0vCl2aoHUYq2FNI", "type": "rectangle", "x": -12030.256565275688, "y": -5850.791446104252, "width": 58.19703760049899, "height": 49.797471958635015, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3d", "roundness": { "type": 3 }, "seed": 1275137682, "version": 310, "versionNonce": 1190127314, "isDeleted": false, "boundElements": [ { "type": "text", "id": "YjDIML30ML_OIagL53AS7" } ], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false }, { "id": "YjDIML30ML_OIagL53AS7", "type": "text", "x": -12014.950076809911, "y": -5835.892710124935, "width": 27.584060668945312, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3e", "roundness": null, "seed": 969239634, "version": 201, "versionNonce": 1504656530, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "text": "Job", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "spNX_P0vCl2aoHUYq2FNI", "originalText": "Job", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "roLE-vulHyb90HkGBeWOg", "type": "arrow", "x": -12257.84467662396, "y": -5827.205450872114, "width": 70.59580406189957, "height": 2.338746917587855, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3f", "roundness": { "type": 2 }, "seed": 850924050, "version": 197, "versionNonce": 226351698, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 70.59580406189957, 2.338746917587855 ] ], "startBinding": { "elementId": "Myv5MZEGhzImarZ4DewhH", "mode": "orbit", "fixedPoint": [ 1.0000000000000004, 0.5001 ] }, "endBinding": { "elementId": "xRfLT-cGKH79iSUz6kUFQ", "mode": "orbit", "fixedPoint": [ 0, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "en04My76pQ8r0S85BhQtt", "type": "text", "x": -12119.41006790419, "y": -5769.839684158622, "width": 106.19219970703125, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3g", "roundness": null, "seed": 1238368210, "version": 143, "versionNonce": 288538642, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "text": "Shared Queue", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Shared Queue", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "tDrplPtPnl0Igs7rBHJO3", "type": "rectangle", "x": -11655.964459104514, "y": -5860.378899487908, "width": 94.85584173908215, "height": 87.05241510590095, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3h", "roundness": { "type": 3 }, "seed": 455357842, "version": 342, "versionNonce": 549776850, "isDeleted": false, "boundElements": [ { "id": "yxRS4ict-AcK4eQA8pgNy", "type": "arrow" }, { "id": "IozgRfCgusmWAj1VxqBv2", "type": "arrow" }, { "id": "o0FV2LZFgtX1l-v7d378d", "type": "arrow" }, { "id": "zn35auLWSG8ZSvr_VDq8h", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false }, { "id": "E2TpBxoT9anq7P_t9Q8d-", "type": "rectangle", "x": -11873.96445440391, "y": -5927.887555834908, "width": 172.1912604447565, "height": 275.98597070430407, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3i", "roundness": { "type": 3 }, "seed": 669758290, "version": 239, "versionNonce": 598330846, "isDeleted": false, "boundElements": [ { "id": "tHbixK6S0pl77GX5HKpv1", "type": "arrow" }, { "id": "yxRS4ict-AcK4eQA8pgNy", "type": "arrow" }, { "id": "1xgJIQUDmc9NN6N3eRgJA", "type": "arrow" }, { "id": "5LqeIHU2zwf8nmwm57OqC", "type": "arrow" }, { "id": "gcGlApM_0FEhblkI-zH0Y", "type": "arrow" } ], "updated": 1794138625277, "created": 1794120896906, "link": null, "locked": false }, { "id": "wBVdsFbqPKI2sQPe40XNP", "type": "text", "x": -11853.13386182516, "y": -5642.845863099361, "width": 134.9122772216797, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3j", "roundness": null, "seed": 847083794, "version": 106, "versionNonce": 1023222098, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "text": "barista-rs worker", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "barista-rs worker", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "tHbixK6S0pl77GX5HKpv1", "type": "arrow", "x": -11953.760426873543, "y": -5828.292611917638, "width": 74.29597246963203, "height": 1.6758242007026638, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3k", "roundness": { "type": 2 }, "seed": 441485010, "version": 247, "versionNonce": 2103078494, "isDeleted": false, "boundElements": [], "updated": 1794138625286, "created": 1794120896906, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 74.29597246963203, 1.6758242007026638 ] ], "startBinding": { "elementId": "xRfLT-cGKH79iSUz6kUFQ", "mode": "inside", "fixedPoint": [ 0.9973753924151665, 0.45652150981594397 ] }, "endBinding": { "elementId": "E2TpBxoT9anq7P_t9Q8d-", "mode": "orbit", "fixedPoint": [ 0.37263533865827, 0.3726353386582698 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "yxRS4ict-AcK4eQA8pgNy", "type": "arrow", "x": -11702.973169573486, "y": -5816.293234783355, "width": 41.50871046897373, "height": 0.47001833825197536, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3l", "roundness": { "type": 2 }, "seed": 939252882, "version": 255, "versionNonce": 49869470, "isDeleted": false, "boundElements": [], "updated": 1794138625288, "created": 1794120896906, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 41.50871046897373, -0.47001833825197536 ] ], "startBinding": { "elementId": "E2TpBxoT9anq7P_t9Q8d-", "mode": "inside", "fixedPoint": [ 0.9930311468117824, 0.40434780350164107 ] }, "endBinding": { "elementId": "tDrplPtPnl0Igs7rBHJO3", "mode": "orbit", "fixedPoint": [ 0.01718207998646879, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "2ihFAh_D1lNiqdn61hGvT", "type": "text", "x": -11657.161560293911, "y": -5765.580453333695, "width": 95.52017211914062, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3m", "roundness": null, "seed": 959871570, "version": 113, "versionNonce": 840698514, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896906, "link": null, "locked": false, "text": "A single lane", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "A single lane", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "_PYVwNRC7ry9tFYmOs43h", "type": "line", "x": -11778.56932626488, "y": -5606.557572231055, "width": 0.9462404289666893, "height": 94.85228290795442, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3n", "roundness": { "type": 2 }, "seed": 2094489618, "version": 97, "versionNonce": 455141458, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0.9462404289666893, 94.85228290795442 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "ZBUmRg7ilXMBx3l1DkvGM", "type": "line", "x": -11284.661751296235, "y": -5614.282023454333, "width": 0.3893148151857986, "height": 41.04100303329926, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3o", "roundness": { "type": 2 }, "seed": 1859489234, "version": 124, "versionNonce": 1396968978, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -0.3893148151857986, 41.04100303329926 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "QQJPQkrMuP46RtYbxMjN3", "type": "rectangle", "x": -11395.105597946522, "y": -5572.781769371636, "width": 260.9867204488042, "height": 172.97903944879027, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3p", "roundness": { "type": 3 }, "seed": 1682441106, "version": 317, "versionNonce": 1335685549, "isDeleted": false, "boundElements": [], "updated": 1794627014965, "created": 1794120896907, "link": null, "locked": false }, { "id": "LXAnHFgFcDu5nPfUbUVW-", "type": "text", "x": -11380.25557518687, "y": -5565.215746707532, "width": 237.04046630859375, "height": 160, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3q", "roundness": null, "seed": 2003157330, "version": 269, "versionNonce": 1436779213, "isDeleted": false, "boundElements": [], "updated": 1794627009513, "created": 1794120896907, "link": null, "locked": false, "text": "- Limited Computing Resources\nare available\n- Varying Computing Resources\nbased on the Jobs\n- Multiple jobs are competing \nfor the shared computing\nresources\n- Jobs cannot be splitted", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "- Limited Computing Resources\nare available\n- Varying Computing Resources\nbased on the Jobs\n- Multiple jobs are competing \nfor the shared computing\nresources\n- Jobs cannot be splitted", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "IozgRfCgusmWAj1VxqBv2", "type": "arrow", "x": -11562.023181335486, "y": -5818.695631313829, "width": 146.056504824865, "height": 17.546495140669492, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3r", "roundness": { "type": 2 }, "seed": 1669711634, "version": 209, "versionNonce": 1906913106, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 146.056504824865, 17.546495140669492 ] ], "startBinding": { "elementId": "tDrplPtPnl0Igs7rBHJO3", "mode": "inside", "fixedPoint": [ 0.9903583801135879, 0.4788295433661463 ] }, "endBinding": { "elementId": "ID8KraUblzU5CxCAPUlHI", "mode": "orbit", "fixedPoint": [ 0, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "o0FV2LZFgtX1l-v7d378d", "type": "arrow", "x": -11562.560368288401, "y": -5780.07047765482, "width": 153.5326046377054, "height": 76.66659231373126, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3s", "roundness": { "type": 2 }, "seed": 1629426898, "version": 234, "versionNonce": 1596683538, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 153.5326046377054, 76.66659231373126 ] ], "startBinding": { "elementId": "tDrplPtPnl0Igs7rBHJO3", "mode": "inside", "fixedPoint": [ 0.984695186966329, 0.9225295097831602 ] }, "endBinding": { "elementId": "mjUQRLG1mG8sMIUYCuO31", "mode": "orbit", "fixedPoint": [ 0, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "KqnNN8qVlwTvMTTzLDDyW", "type": "text", "x": -12259.95096484676, "y": -7211.749134428242, "width": 635.3184750257684, "height": 84.9501201521457, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3t", "roundness": null, "seed": 29928082, "version": 416, "versionNonce": 90276577, "isDeleted": false, "boundElements": [], "updated": 1794255163283, "created": 1794120896907, "link": null, "locked": false, "text": "BRAINSTORMING", "fontSize": 67.96009612171656, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "BRAINSTORMING", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "LWrtiFecu-RYrVDJWRkrv", "type": "rectangle", "x": -12059.469719090013, "y": -5506.7968558818075, "width": 601.962732687349, "height": 738.4606658220947, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3u", "roundness": { "type": 3 }, "seed": 205097042, "version": 573, "versionNonce": 1448794258, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false }, { "id": "55K0A-rpBLZVMssQlrmcH", "type": "text", "x": -12041.778329746356, "y": -5486.710530044594, "width": 571.2040447168779, "height": 718.5447931334047, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3v", "roundness": null, "seed": 1481158162, "version": 1615, "versionNonce": 1603309219, "isDeleted": false, "boundElements": [], "updated": 1794624657548, "created": 1794120896907, "link": null, "locked": false, "text": "## KEY FEATURES:\n0. Fault tolerant ( with dispatcher replicas and all the\nknown edge cases handled )\n1. Fairness: SFQ + Penalization/Decay + Per Tenant\nConcurrency Cap\n2. Atomic Dispatch: 1 transaction = lowest start tag pick\n+ mark it running in the DB + update the virtual_now +\nvirtual_final_tag \n3. Idempotent\n4. Job Stealing: Smart enough to dispatch the sm jobs to\nmd or lg resource i.e the job asks if it is big enough to run\nthe resource ? if the answer is yes & the container is idle\nand no jobs of that size is queued then runs the job on\nthat resource.\nPrefer-Fit policy: A container first looks for the lowest\ntag among jobs that need exactly its size, and only steals\ndownward when there are none. Gives lg-only jobs priority\non lg at a small cost to global tag order.\n5. Job Cancellation\n6. Prevents Starvation\n7. Weighted Priority\n8. Optional Preemption\n9. Penalization: The scheduler lowers the priority of \nsustained heavy users #History-aware-fairness. The\npenalty decreases with time\n10. Per Tenant Concurrency Cap: If all containers/slots\nare occupied by Tenant A, then Tenant B has to wait for\nthe slot to open regardless of the queue order, Thus a\ntenant can only hog/use a capped slots concurrently", "fontSize": 19.82192532781806, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "## KEY FEATURES:\n0. Fault tolerant ( with dispatcher replicas and all the known edge cases handled )\n1. Fairness: SFQ + Penalization/Decay + Per Tenant Concurrency Cap\n2. Atomic Dispatch: 1 transaction = lowest start tag pick + mark it running in the DB + update the virtual_now + virtual_final_tag \n3. Idempotent\n4. Job Stealing: Smart enough to dispatch the sm jobs to md or lg resource i.e the job asks if it is big enough to run the resource ? if the answer is yes & the container is idle and no jobs of that size is queued then runs the job on that resource.\nPrefer-Fit policy: A container first looks for the lowest tag among jobs that need exactly its size, and only steals downward when there are none. Gives lg-only jobs priority on lg at a small cost to global tag order.\n5. Job Cancellation\n6. Prevents Starvation\n7. Weighted Priority\n8. Optional Preemption\n9. Penalization: The scheduler lowers the priority of \nsustained heavy users #History-aware-fairness. The penalty decreases with time\n10. Per Tenant Concurrency Cap: If all containers/slots\nare occupied by Tenant A, then Tenant B has to wait for the slot to open regardless of the queue order, Thus a tenant can only hog/use a capped slots concurrently", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "F9B6BZCtcSCZUeb3S4vX-", "type": "line", "x": -11999.431907262142, "y": -5460.977992436824, "width": 166.71695569629662, "height": 1.2726172980114825, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3w", "roundness": { "type": 2 }, "seed": 720182226, "version": 136, "versionNonce": 346512402, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 166.71695569629662, -1.2726172980114825 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "cnFfKXfYOdfHo3TW3suoN", "type": "line", "x": -12297.986355706553, "y": -5465.357643140038, "width": 1.2726172980115962, "height": 120.9016144292732, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3x", "roundness": { "type": 2 }, "seed": 306041234, "version": 87, "versionNonce": 54921682, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1.2726172980115962, 120.9016144292732 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "akO7eEL-7Xc-ORao07WTw", "type": "rectangle", "x": -12451.746228662661, "y": -5336.589612223253, "width": 325.79803283286634, "height": 391.0806182140424, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3y", "roundness": { "type": 3 }, "seed": 1097977682, "version": 175, "versionNonce": 2018711449, "isDeleted": false, "boundElements": [], "updated": 1794130970762, "created": 1794120896907, "link": null, "locked": false }, { "id": "x1LgcHdDhdhI_letWAqDo", "type": "text", "x": -12429.49448424997, "y": -5313.17698697442, "width": 278.0145300156683, "height": 500.0799397403704, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b3z", "roundness": null, "seed": 1826624786, "version": 656, "versionNonce": 124576496, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794120896907, "link": null, "locked": false, "text": "Carries information (v5):\n\n- unique ID\n- tenant\n- tier (weight comes from\n the tier, not the job)\n- job_type + est_cost\n (CPU seconds)\n- target resource: sm/md/lg\n- enqueue_seq\n- start_tag (set at dispatch)\n- attempt (fencing token)\n- failures (vs max_attempts)\n- idempotency_key\n- timeout_at ( this is a job\ntimeout not worker timeout, \nthat is handled through\nlease_timeout\nconfigured through the\ndispatcher)\n- (finish_tag is per tenant)", "fontSize": 19.05066437106173, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Carries information (v5):\n\n- unique ID\n- tenant\n- tier (weight comes from\n the tier, not the job)\n- job_type + est_cost\n (CPU seconds)\n- target resource: sm/md/lg\n- enqueue_seq\n- start_tag (set at dispatch)\n- attempt (fencing token)\n- failures (vs max_attempts)\n- idempotency_key\n- timeout_at ( this is a job\ntimeout not worker timeout, \nthat is handled through\nlease_timeout\nconfigured through the\ndispatcher)\n- (finish_tag is per tenant)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "SNoDrfPBBafdS4lRnwNPx", "type": "rectangle", "x": -13086.94459915956, "y": -6067.335779063168, "width": 428.3453647900501, "height": 476.72653717471934, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b40", "roundness": { "type": 3 }, "seed": 637510354, "version": 491, "versionNonce": 1165476626, "isDeleted": false, "boundElements": [ { "id": "pCi_eGr_nvYIvpW9aV1wa", "type": "arrow" }, { "id": "Pf8_2s_TVA6fmrB0HGymc", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false }, { "id": "6ZDnTsac2S5wZo69TUiHw", "type": "text", "x": -13076.215451078682, "y": -6040.225126499224, "width": 412.0310109764948, "height": 443.18675548708654, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b41", "roundness": null, "seed": 929527954, "version": 562, "versionNonce": 728862930, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "text": "SAAS Application\n\n- Multi-tenant\n- Tiered Subscription:\n - Free\n - Bronze\n - Silver\n - Gold\n - Diamond\n\n\n", "fontSize": 29.545783699139104, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "SAAS Application\n\n- Multi-tenant\n- Tiered Subscription:\n - Free\n - Bronze\n - Silver\n - Gold\n - Diamond\n\n\n", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "z-Tl7UcBvTM4DMyaA3F1Z", "type": "diamond", "x": -11898.676368098812, "y": -6113.249034704056, "width": 176.63791092407752, "height": 155.6707716099221, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b42", "roundness": { "type": 2 }, "seed": 13538898, "version": 513, "versionNonce": 1416341138, "isDeleted": false, "boundElements": [ { "id": "0ncHVm1R2j8BS1j9L3roQ", "type": "arrow" }, { "id": "bP3DQCJAyCmztnJp9ANrj", "type": "arrow" }, { "id": "rf0nnm0CW6xH3Aae8JBvK", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false }, { "id": "ZKk6tLghaQqWdSCRLl4qf", "type": "text", "x": -11830.842833888972, "y": -6092.482829936253, "width": 45.745956764135826, "height": 36.73787796845653, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b43", "roundness": null, "seed": 1393294354, "version": 142, "versionNonce": 152896594, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "text": "virtual\nclock", "fontSize": 14.69515118738262, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "virtual\nclock", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "-PcYFs30QvbmmH71VBKEQ", "type": "line", "x": -11807.697472618947, "y": -5962.718936744122, "width": 2.761168961170142, "height": 34.475722788127655, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b44", "roundness": { "type": 2 }, "seed": 1276553682, "version": 116, "versionNonce": 1667017234, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 2.761168961170142, 34.475722788127655 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "LcXtHh8wMu2EvjDLwMeHW", "type": "rectangle", "x": -12494.536359714466, "y": -6132.712451222374, "width": 224.06817098635702, "height": 78.81304814365336, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b45", "roundness": { "type": 3 }, "seed": 856006546, "version": 331, "versionNonce": 706844626, "isDeleted": false, "boundElements": [ { "id": "JdHRZvlEKC_nhsiDk6jVj", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false }, { "id": "Dv5aYTMcDjOpDpTXMCEap", "type": "text", "x": -12469.20620666133, "y": -6109.630591043866, "width": 169.05606079101562, "height": 45, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b46", "roundness": null, "seed": 1410325842, "version": 107, "versionNonce": 763547026, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "text": "Database", "fontSize": 36, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Database", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "SLGvRabX7VwLAnhi1hEHR", "type": "line", "x": -12389.369945191296, "y": -6138.278542606682, "width": 193.44494705377292, "height": 146.91773452727227, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b47", "roundness": { "type": 2 }, "seed": 1552704274, "version": 191, "versionNonce": 582437714, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -193.44494705377292, -146.91773452727227 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "m1rThvrEsSw-cr8cjmiLn", "type": "rectangle", "x": -13326.234100431797, "y": -6408.520473964665, "width": 736.4713446099908, "height": 237.96361189754717, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b48", "roundness": { "type": 3 }, "seed": 921841874, "version": 258, "versionNonce": 1446162, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false }, { "id": "RORq7t_bUGj4gBzyOz-EU", "type": "text", "x": -13310.837385038893, "y": -6395.57450895394, "width": 690.3402286308867, "height": 207.91569817543592, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b49", "roundness": null, "seed": 1725449874, "version": 460, "versionNonce": 1494621906, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "text": "- Persistence\n- Job Cancellation: Why not Kafka as the single\nsource of truth because: once the jobs are\ndispatched to Kafka, cannot be cancelled. Thus a\nDB is a right way to go about in this setup.\n- Crash Recovery", "fontSize": 27.72209309005812, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "- Persistence\n- Job Cancellation: Why not Kafka as the single source of truth because: once the jobs are dispatched to Kafka, cannot be cancelled. Thus a DB is a right way to go about in this setup.\n- Crash Recovery", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "eXI98ZEpQXChhpPqVEhmv", "type": "rectangle", "x": -11416.40702495152, "y": -5963.747991288424, "width": 320.13482088244155, "height": 101.83858421844624, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4A", "roundness": { "type": 3 }, "seed": 1693819986, "version": 251, "versionNonce": 978220178, "isDeleted": false, "boundElements": [ { "id": "5LqeIHU2zwf8nmwm57OqC", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false }, { "id": "oje7ZA8v4Z7k7Hr89Lhkk", "type": "rectangle", "x": -11420.092813475785, "y": -5853.557092436359, "width": 328.55944731096434, "height": 91.16329581759442, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4B", "roundness": { "type": 3 }, "seed": 2065779218, "version": 221, "versionNonce": 91624018, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false }, { "id": "zxGbFiEPxx4mJADSt3kgs", "type": "rectangle", "x": -11413.155752682129, "y": -5749.46067659373, "width": 323.82060940670567, "height": 85.64772813507905, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4C", "roundness": { "type": 3 }, "seed": 542259154, "version": 243, "versionNonce": 651050818, "isDeleted": false, "boundElements": [ { "id": "gcGlApM_0FEhblkI-zH0Y", "type": "arrow" } ], "updated": 1794138625277, "created": 1794120896907, "link": null, "locked": false }, { "id": "KOfV7M4-1TpGjeU4CIsow", "type": "line", "x": -11025.7161601952, "y": -5984.592638967128, "width": 13.69004686992048, "height": 326.9798298556162, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4D", "roundness": { "type": 2 }, "seed": 1419734418, "version": 168, "versionNonce": 839853522, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 13.69004686992048, 326.9798298556162 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "TPLbk6a34zdLEO3BBnM3V", "type": "arrow", "x": -11026.769267422336, "y": -5984.0661142771305, "width": 106.88711519081289, "height": 1.0530783035653712, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4E", "roundness": { "type": 2 }, "seed": 2101341010, "version": 76, "versionNonce": 1015978898, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896907, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -106.88711519081289, -1.0530783035653712 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "hj2NK9DE6SNZ19DKslH17", "type": "arrow", "x": -11012.55275370956, "y": -5658.665887415077, "width": 102.72981940920954, "height": 10.54618332644975, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4F", "roundness": { "type": 2 }, "seed": 1082831122, "version": 108, "versionNonce": 991517010, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -102.72981940920954, 10.54618332644975 ] ], "startBinding": null, "endBinding": { "elementId": "oJ2a_LxQRXJru3pENR3Ch", "mode": "orbit", "fixedPoint": [ 0.5062744986143884, 0.5062744986143883 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "Oc1csImLVNgHQ_Bbdz9uI", "type": "text", "x": -10990.43813825826, "y": -5845.4498404926435, "width": 262.999755859375, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4G", "roundness": null, "seed": 1643560658, "version": 105, "versionNonce": 486605785, "isDeleted": false, "boundElements": [], "updated": 1794131086697, "created": 1794120896908, "link": null, "locked": false, "text": "Different Consumer Groups\n( Pull based workers )", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Different Consumer Groups\n( Pull based workers )", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Qk7rPgJZrdw5N2wFHwdJG", "type": "text", "x": -11401.34870087696, "y": -5963.603018850648, "width": 176.2198486328125, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4H", "roundness": null, "seed": 505091218, "version": 63, "versionNonce": 1322036434, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "Consumer Group A", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Consumer Group A", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "AJGZlXUaZHp--ZZyc8fj7", "type": "text", "x": -11401.921350539458, "y": -5854.808877603013, "width": 177.9198455810547, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4I", "roundness": null, "seed": 112112210, "version": 66, "versionNonce": 369312402, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "Consumer Group B", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Consumer Group B", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "UOy1IPpNVsAhmnisxlyKN", "type": "text", "x": -11403.639110803562, "y": -5751.837937133442, "width": 175.27984619140625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4J", "roundness": null, "seed": 1586010130, "version": 86, "versionNonce": 1835081810, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "Consumer Group C", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Consumer Group C", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "XPpkSNfWrvFQwHdZarDHe", "type": "line", "x": -11322.221329859107, "y": -5890.794408919799, "width": 130.35224711052024, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4K", "roundness": { "type": 2 }, "seed": 1973405138, "version": 127, "versionNonce": 1822972738, "isDeleted": false, "boundElements": [], "updated": 1794138644889, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 130.35224711052024, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "HlAw1G1oNkLLB_Pq1a-DF", "type": "line", "x": -11319.836849028958, "y": -5792.872827572871, "width": 130.35217052805046, "height": 1.3941072796449134, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4L", "roundness": { "type": 2 }, "seed": 84622226, "version": 131, "versionNonce": 1941597122, "isDeleted": false, "boundElements": [], "updated": 1794138639337, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 130.35217052805046, -1.3941072796449134 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "14CtqmV8K09DpEXhbn8Wq", "type": "line", "x": -11308.6837116619, "y": -5684.331670929948, "width": 121.29032004541978, "height": 4.18243671263906, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4M", "roundness": { "type": 2 }, "seed": 184678738, "version": 120, "versionNonce": 1585279362, "isDeleted": false, "boundElements": [], "updated": 1794138651294, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 121.29032004541978, -4.18243671263906 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "5RN8dE9byZQpjRbRVDMsU", "type": "rectangle", "x": -11189.652589229501, "y": -5938.035781913529, "width": 87.76312163726138, "height": 60.004214960299514, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4N", "roundness": { "type": 3 }, "seed": 651886354, "version": 168, "versionNonce": 1712154450, "isDeleted": false, "boundElements": [ { "type": "text", "id": "wMClFx_9zxR02IZ-3LiKJ" } ], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false }, { "id": "wMClFx_9zxR02IZ-3LiKJ", "type": "text", "x": -11177.331094634015, "y": -5928.033674433379, "width": 63.12013244628906, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4O", "roundness": null, "seed": 1963805906, "version": 114, "versionNonce": 709405970, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "sm\nlim: 3GB", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "5RN8dE9byZQpjRbRVDMsU", "originalText": "sm\nlim: 3GB", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Om7Dc2jIxyOiamg7gGENJ", "type": "rectangle", "x": -11188.542889347498, "y": -5837.2165527375555, "width": 91.12015099183466, "height": 63.30454286791064, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4P", "roundness": { "type": 3 }, "seed": 1798300306, "version": 149, "versionNonce": 1519729362, "isDeleted": false, "boundElements": [ { "type": "text", "id": "MPCSGDAGyguAA2ISTvKai" } ], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false }, { "id": "MPCSGDAGyguAA2ISTvKai", "type": "text", "x": -11174.798876778827, "y": -5825.5642813036, "width": 63.63212585449219, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4Q", "roundness": null, "seed": 872005714, "version": 81, "versionNonce": 1971982482, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "md\nlim: 6GB", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "Om7Dc2jIxyOiamg7gGENJ", "originalText": "md\nlim: 6GB", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "zgx_0Jfgv_zuNvqsJ0ZVJ", "type": "rectangle", "x": -11188.805024586276, "y": -5735.1821617897795, "width": 93.03852874903463, "height": 61.38621779892139, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4R", "roundness": { "type": 3 }, "seed": 188868114, "version": 192, "versionNonce": 249441874, "isDeleted": false, "boundElements": [ { "type": "text", "id": "WC5zPEu6Ydc2oArCNYqS6" } ], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false }, { "id": "WC5zPEu6Ydc2oArCNYqS6", "type": "text", "x": -11181.19783906918, "y": -5724.489052890319, "width": 77.82415771484375, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4S", "roundness": null, "seed": 436163538, "version": 106, "versionNonce": 728273938, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "lg\nlim: 12 GB", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "zgx_0Jfgv_zuNvqsJ0ZVJ", "originalText": "lg\nlim: 12 GB", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Z9Kbym_Qek9z_dG8l5TgX", "type": "line", "x": -11287.812283953266, "y": -5967.644589571404, "width": 1.55819617742236, "height": 61.54686599164893, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4T", "roundness": { "type": 2 }, "seed": 1380518290, "version": 84, "versionNonce": 488653266, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -1.55819617742236, -61.54686599164893 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "6oPIPY2hZHrlVOxiaB2ZE", "type": "rectangle", "x": -11438.173635920679, "y": -6098.413908775195, "width": 329.54843450470275, "height": 66.10614644895719, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4U", "roundness": { "type": 3 }, "seed": 1263929170, "version": 273, "versionNonce": 217601758, "isDeleted": false, "boundElements": [], "updated": 1794138898299, "created": 1794120896908, "link": null, "locked": false }, { "id": "ZAfCBLPoLaD6DtRbtbJOw", "type": "text", "x": -11424.121959410606, "y": -6089.754545093384, "width": 291.459716796875, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4V", "roundness": null, "seed": 494750994, "version": 249, "versionNonce": 32315486, "isDeleted": false, "boundElements": [], "updated": 1794138901691, "created": 1794120896908, "link": null, "locked": false, "text": "- Per Tenant Concurrency Cap\nE.g: tenant A: 1/3 slots used", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "- Per Tenant Concurrency Cap\nE.g: tenant A: 1/3 slots used", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "KSwenXBfUz3o9L-4MAWYL", "type": "rectangle", "x": -10622.319304654424, "y": -6357.873844147513, "width": 748.6123421216155, "height": 1622.028975308738, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4W", "roundness": { "type": 3 }, "seed": 1043716818, "version": 472, "versionNonce": 334369310, "isDeleted": false, "boundElements": [], "updated": 1794139748600, "created": 1794120896908, "link": null, "locked": false }, { "id": "dSGq1XlrTvPtq4a2aW9Y4", "type": "text", "x": -10599.596350046822, "y": -6319.545000873102, "width": 692.0868153301317, "height": 1500.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4X", "roundness": null, "seed": 1544990866, "version": 2280, "versionNonce": 673671310, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794120896908, "link": null, "locked": false, "text": "## Key Decisions taken along the way: \n\n1. Single lane instead of multiple lanes for each consumer group: I have\ntaken this decision because:\n A. To enable job stealing: can direct to md or lg resource for\nsm jobs if sm resource is not idle but the other resources are !, if not\nfor single lane we could not direct it to cross consumer groups\n B. To allow per tenant concurrency cap across cross consumer\ngroups, if not for single lane we would have to limit ourselves to a\nsingle consumer group only\n\n2. Scheduling algorithm: SFQ over DRR(Deficit Round Robin)\n3. The virtual_final_tag is an estimated time taken to complete the\njob and not the actuall time. \n4. (v5) Ties in start time: higher tier first, then lower enqueue_seq.\n5. Job Stealing: Here we have 2 options to perform job stealing:\n A. Prefer Fit policy: First see if lg jobs are present in the queue,\nonly steal downward if there are none\n B. Reserve Policy: Only steal job if stolen job's tag is below the\nlowest lg only tag by some margin\n\nI have decided to go with the Prefer Fit policy although it steals all\nthe sm jobs unlike the reserve policy but we have to add additional\nrule such that if lg job appear in the queue then that would\nautomatically take the precedence over other jobs.\n6. (v5) Provisional finish_tag: tags are PER TENANT. A job gets its\nstart_tag at dispatch = GREATEST(virtual_now, tenant.finish_tag).\nOn complete, correct ONE value:\ntenant.finish_tag += (actual_cpu - est_cost) / weight.\nNo queued job is updated, so no race occurs.\n7. Charge in CPU seconds, not wall seconds. This is the catch with\nmixed container sizes. A job that takes 40 wall seconds on sm takes\nmaybe 10 on lg, and if you bill wall time, running on lg becomes a\ndiscount. Bill what the cgroup reports as CPU time consumed.\n8. One container run one job at a time (worker = container = one\nexecution slot):\n A) No need for a separate cgroup for each job, each container has\nit's own cgroup\n B) Clean kill with docker stop\n C) Isolation: A job that uses too much CPU hits only it's own\ncontainer limit\n9. gRPC over HTTP : \n A) Cancel while running. The dispatcher must tell a busy worker to\nstop. With gRPC, one bidirectional stream per worker carries\nheartbeats up and Cancel down at once. With HTTP, the worker has\nto poll, or you put a cancel: true flag in the heartbeat response. That\nworks, but a cancel then takes up to one heartbeat interval to arrive.\n B) (v5) Lease and heartbeat. A broken stream shows that the\nCONNECTION stopped, not that the worker stopped. It is a warning, not\na verdict: wait lease_timeout before requeue.\n C) Lease handout. A worker opens a stream and the dispatcher\npushes a job to it when one is ready. You don\u2019t need a polling loop for\nlease(size).\n D) Typed contract. A .proto file fixes ResourceClass, JobId and\ncpu_seconds for every worker language.\n10. (v5) Dispatch concurrency: singleton lock on scheduler_state.\nOne dispatch at a time across all replicas. No SKIP LOCKED.\n11. (v5) Zombie policy: self-kill + fencing (worker_self_kill_after\n< lease_timeout). The database still rejects stale attempts.\n12. (v5) Transport: one gRPC bidirectional stream per worker.", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "## Key Decisions taken along the way: \n\n1. Single lane instead of multiple lanes for each consumer group: I have\ntaken this decision because:\n A. To enable job stealing: can direct to md or lg resource for\nsm jobs if sm resource is not idle but the other resources are !, if not\nfor single lane we could not direct it to cross consumer groups\n B. To allow per tenant concurrency cap across cross consumer\ngroups, if not for single lane we would have to limit ourselves to a\nsingle consumer group only\n\n2. Scheduling algorithm: SFQ over DRR(Deficit Round Robin)\n3. The virtual_final_tag is an estimated time taken to complete the\njob and not the actuall time. \n4. (v5) Ties in start time: higher tier first, then lower enqueue_seq.\n5. Job Stealing: Here we have 2 options to perform job stealing:\n A. Prefer Fit policy: First see if lg jobs are present in the queue,\nonly steal downward if there are none\n B. Reserve Policy: Only steal job if stolen job's tag is below the\nlowest lg only tag by some margin\n\nI have decided to go with the Prefer Fit policy although it steals all\nthe sm jobs unlike the reserve policy but we have to add additional\nrule such that if lg job appear in the queue then that would\nautomatically take the precedence over other jobs.\n6. (v5) Provisional finish_tag: tags are PER TENANT. A job gets its\nstart_tag at dispatch = GREATEST(virtual_now, tenant.finish_tag).\nOn complete, correct ONE value:\ntenant.finish_tag += (actual_cpu - est_cost) / weight.\nNo queued job is updated, so no race occurs.\n7. Charge in CPU seconds, not wall seconds. This is the catch with\nmixed container sizes. A job that takes 40 wall seconds on sm takes\nmaybe 10 on lg, and if you bill wall time, running on lg becomes a\ndiscount. Bill what the cgroup reports as CPU time consumed.\n8. One container run one job at a time (worker = container = one\nexecution slot):\n A) No need for a separate cgroup for each job, each container has\nit's own cgroup\n B) Clean kill with docker stop\n C) Isolation: A job that uses too much CPU hits only it's own\ncontainer limit\n9. gRPC over HTTP : \n A) Cancel while running. The dispatcher must tell a busy worker to\nstop. With gRPC, one bidirectional stream per worker carries\nheartbeats up and Cancel down at once. With HTTP, the worker has\nto poll, or you put a cancel: true flag in the heartbeat response. That\nworks, but a cancel then takes up to one heartbeat interval to arrive.\n B) (v5) Lease and heartbeat. A broken stream shows that the\nCONNECTION stopped, not that the worker stopped. It is a warning, not\na verdict: wait lease_timeout before requeue.\n C) Lease handout. A worker opens a stream and the dispatcher\npushes a job to it when one is ready. You don\u2019t need a polling loop for\nlease(size).\n D) Typed contract. A .proto file fixes ResourceClass, JobId and\ncpu_seconds for every worker language.\n10. (v5) Dispatch concurrency: singleton lock on scheduler_state.\nOne dispatch at a time across all replicas. No SKIP LOCKED.\n11. (v5) Zombie policy: self-kill + fencing (worker_self_kill_after\n< lease_timeout). The database still rejects stale attempts.\n12. (v5) Transport: one gRPC bidirectional stream per worker.", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Cd29O3OH9U6_HSlaHkw-C", "type": "rectangle", "x": -11646.585278094044, "y": -5854.149790596666, "width": 76.00211559976333, "height": 73.7955281222944, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4Y", "roundness": { "type": 3 }, "seed": 987156050, "version": 399, "versionNonce": 1726144146, "isDeleted": false, "boundElements": [ { "type": "text", "id": "kk1H5PBiZDBErZAdITJxT" } ], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false }, { "id": "kk1H5PBiZDBErZAdITJxT", "type": "text", "x": -11622.376250628635, "y": -5827.252026535519, "width": 27.584060668945312, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4Z", "roundness": null, "seed": 256717842, "version": 290, "versionNonce": 938294354, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "Job", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "Cd29O3OH9U6_HSlaHkw-C", "originalText": "Job", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "zn35auLWSG8ZSvr_VDq8h", "type": "arrow", "x": -11556.970141027634, "y": -5848.983022687936, "width": 141.09738286555012, "height": 50.61652637631357, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4a", "roundness": { "type": 2 }, "seed": 1534124498, "version": 308, "versionNonce": 996532754, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 141.09738286555012, -50.61652637631357 ] ], "startBinding": { "elementId": "tDrplPtPnl0Igs7rBHJO3", "mode": "orbit", "fixedPoint": [ 0.990148905339841, 0.15181301659540528 ] }, "endBinding": { "elementId": "WQNi7orzkmZF94NxStNjF", "mode": "orbit", "fixedPoint": [ 0, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "J819DX7bg7RSIJ53Gw_2V", "type": "line", "x": -12376.279538519813, "y": -6053.390316805457, "width": 505.0165135079841, "height": 143.49117751518298, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4b", "roundness": { "type": 2 }, "seed": 1247336338, "version": 278, "versionNonce": 474206162, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 57.723022092476185, 133.85097009210455 ], [ 505.0165135079841, 143.49117751518298 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "sGSFchftMrkaiPMbsT4sK", "type": "rectangle", "x": -11864.404441268342, "y": -5922.419427989611, "width": 154.2653903986727, "height": 126.50720732925132, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4c", "roundness": { "type": 3 }, "seed": 205264210, "version": 159, "versionNonce": 245237138, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false }, { "id": "exMpLGeyqAsqq5duq1NX3", "type": "rectangle", "x": -11865.719126004178, "y": -5790.652424002338, "width": 153.32477108926443, "height": 127.9273780119396, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4d", "roundness": { "type": 3 }, "seed": 846306066, "version": 120, "versionNonce": 732660562, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false }, { "id": "ouiO5fNpn-NIYMV-3NSGT", "type": "text", "x": -11848.669056311224, "y": -5919.174855836053, "width": 91.13992309570312, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4e", "roundness": null, "seed": 191916242, "version": 87, "versionNonce": 1304643858, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "Scheduler", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Scheduler", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "rIrsfooBDMTkz_P8vKQw2", "type": "text", "x": -11850.547979444818, "y": -5785.7264226854895, "width": 105.05989074707031, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4f", "roundness": null, "seed": 2127527570, "version": 75, "versionNonce": 1817990866, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "Dispatcher", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Dispatcher", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Oynkp-7ocaTJeoNJjx6Cz", "type": "rectangle", "x": -10642.520374389247, "y": -6741.799121884007, "width": 787.9834690957853, "height": 357.9402988340908, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4g", "roundness": { "type": 3 }, "seed": 1766186066, "version": 260, "versionNonce": 1442083074, "isDeleted": false, "boundElements": [], "updated": 1794139784497, "created": 1794120896908, "link": null, "locked": false }, { "id": "Og4GmyZdDlOFXIfItIe6i", "type": "text", "x": -10614.574782655543, "y": -6706.266636600167, "width": 751.1593017578125, "height": 325.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4h", "roundness": null, "seed": 2024440338, "version": 596, "versionNonce": 1596840320, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794120896908, "link": null, "locked": false, "text": "## Configurable\n\n1. Per Tenant Concurrency cap\n2. Number of consumer groups\n3. Tiered subscriptions hierarchy \n4. Configurable Scheduler: Default SFQ ; DRR configurable (Backloged !!! not\nin the scope of the current implementation)\n5. Steal policy (v5 default: Prefer-Fit). Reserve{M} is optional:\n\"how eager is the resource to steal the downward job\"\n6. timeout_at\n7. lease_timeout (30 s), worker_self_kill_after (25 s)\n8. Containers per group (1 job each), max_attempts\n9. assign_ack_timeout, cancel_grace, max_queued_per_tenant", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "## Configurable\n\n1. Per Tenant Concurrency cap\n2. Number of consumer groups\n3. Tiered subscriptions hierarchy \n4. Configurable Scheduler: Default SFQ ; DRR configurable (Backloged !!! not\nin the scope of the current implementation)\n5. Steal policy (v5 default: Prefer-Fit). Reserve{M} is optional:\n\"how eager is the resource to steal the downward job\"\n6. timeout_at\n7. lease_timeout (30 s), worker_self_kill_after (25 s)\n8. Containers per group (1 job each), max_attempts\n9. assign_ack_timeout, cancel_grace, max_queued_per_tenant", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "cva28bd8UaIkVLQw6q4AA", "type": "text", "x": -11857.063355936769, "y": -5866.7971467957, "width": 144.4802703857422, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4i", "roundness": null, "seed": 133459922, "version": 184, "versionNonce": 515190802, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "text": "SFQ: Earlier start\ntime given priority.", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "SFQ: Earlier start\ntime given priority.", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "qRDUU6pE9mB7-B-JpUPQm", "type": "line", "x": -11859.408104269842, "y": -5849.37206389593, "width": 37.12540376537959, "height": 0.3907949666690911, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4j", "roundness": { "type": 2 }, "seed": 2067441042, "version": 115, "versionNonce": 880369106, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 37.12540376537959, -0.3907949666690911 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "H81dEyc-41iUImyDpINC2", "type": "line", "x": -11859.017320036644, "y": -5845.85491992938, "width": 36.73461953218202, "height": 0.7815899333381822, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4k", "roundness": { "type": 2 }, "seed": 464790354, "version": 92, "versionNonce": 1693569938, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 36.73461953218202, -0.7815899333381822 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "tIHJtjh91P4QN9pWw37Go", "type": "rectangle", "x": -12086.88894789136, "y": -6386.7115149011, "width": 260.7220647955026, "height": 204.19003173235464, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4l", "roundness": { "type": 3 }, "seed": 1941432594, "version": 356, "versionNonce": 1649774930, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false }, { "id": "qH3WPSBjK20Uwb03dn7t8", "type": "rectangle", "x": -11784.997314443443, "y": -6393.02283933305, "width": 247.1933564369076, "height": 201.77187367408013, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4m", "roundness": { "type": 3 }, "seed": 717027026, "version": 257, "versionNonce": 2147194642, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false }, { "id": "0ncHVm1R2j8BS1j9L3roQ", "type": "arrow", "x": -11808.2735295269, "y": -6107.159419946177, "width": 54.821292674659844, "height": 81.56956450390523, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4n", "roundness": { "type": 2 }, "seed": 1633307794, "version": 526, "versionNonce": 240837842, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896908, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 54.821292674659844, -81.56956450390523 ] ], "startBinding": { "elementId": "z-Tl7UcBvTM4DMyaA3F1Z", "mode": "inside", "fixedPoint": [ 0.5117974850300939, 0.03911854932625264 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "du-4LlgMNfMWuDp0wPDTo", "type": "text", "x": -12073.612589778784, "y": -6370.270420026328, "width": 231.18446350097656, "height": 80, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4o", "roundness": null, "seed": 222751314, "version": 157, "versionNonce": 923433618, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "start_tag\n- Per Tenant tag\n- The SFQ runs the job with \nthe smallest tag", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "start_tag\n- Per Tenant tag\n- The SFQ runs the job with \nthe smallest tag", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "fzEzjcvADq4Pr74R2ASjB", "type": "text", "x": -11765.791726374222, "y": -6376.667182954321, "width": 224.25643920898438, "height": 160, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4p", "roundness": null, "seed": 1431949330, "version": 290, "versionNonce": 1489066066, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "virtual_finish_tag\n- Universal tag\n- the estimated time the job\ncompletes at\n- needed just to calculate \nthe next job's start_tag\n- lifetime: until the next job\nis submitted", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "virtual_finish_tag\n- Universal tag\n- the estimated time the job\ncompletes at\n- needed just to calculate \nthe next job's start_tag\n- lifetime: until the next job\nis submitted", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "dXGzDlSPcPa_o4wREcLn4", "type": "text", "x": -12073.030993275896, "y": -6288.469143420513, "width": 234.00045776367188, "height": 120, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4q", "roundness": null, "seed": 1837759954, "version": 194, "versionNonce": 1666612754, "isDeleted": false, "boundElements": [ { "id": "bP3DQCJAyCmztnJp9ANrj", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "- considers the tenants \nhierarchy/priority/weights\nto initialise it\n- System idle: init 0\n- lifetime: until job dispatched\n", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "- considers the tenants \nhierarchy/priority/weights\nto initialise it\n- System idle: init 0\n- lifetime: until job dispatched\n", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "bP3DQCJAyCmztnJp9ANrj", "type": "arrow", "x": -11811.281498081997, "y": -6106.240203944322, "width": 26.06041997398927, "height": 56.99702662574333, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4r", "roundness": { "type": 2 }, "seed": 1928229778, "version": 368, "versionNonce": 868281298, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -26.06041997398927, -56.99702662574333 ] ], "startBinding": { "elementId": "z-Tl7UcBvTM4DMyaA3F1Z", "mode": "inside", "fixedPoint": [ 0.4947684761420104, 0.04502342146344684 ] }, "endBinding": { "elementId": "dXGzDlSPcPa_o4wREcLn4", "mode": "orbit", "fixedPoint": [ 0.9960710075247268, 0.996071007524728 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "5Y_otS1FPOgsYaVdkkPEe", "type": "rectangle", "x": -12069.37979327608, "y": -6531.905838615946, "width": 515.028592672503, "height": 128.70046498962148, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4s", "roundness": { "type": 3 }, "seed": 1203646802, "version": 172, "versionNonce": 1880596882, "isDeleted": false, "boundElements": [ { "id": "rf0nnm0CW6xH3Aae8JBvK", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false }, { "id": "rf0nnm0CW6xH3Aae8JBvK", "type": "arrow", "x": -11810.258624737575, "y": -6113.862696532815, "width": 1.5258040379936801, "height": 283.84267709351013, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4t", "roundness": { "type": 2 }, "seed": 1867527954, "version": 377, "versionNonce": 1670487890, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -1.5258040379936801, -283.84267709351013 ] ], "startBinding": { "elementId": "z-Tl7UcBvTM4DMyaA3F1Z", "mode": "orbit", "fixedPoint": [ 0.5007272449436682, 0.031521228230097266 ] }, "endBinding": { "elementId": "5Y_otS1FPOgsYaVdkkPEe", "mode": "orbit", "fixedPoint": [ 0.5001, 1 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "nNApcfU8fPTAmRr-TVCZD", "type": "text", "x": -12047.380914412925, "y": -6527.474132408732, "width": 485.3289489746094, "height": 100.00000000000001, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4u", "roundness": null, "seed": 1780188370, "version": 307, "versionNonce": 1097127994, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794120896909, "link": null, "locked": false, "text": "virtual_now (v5)\n- Universal tag, one row in scheduler_state\n- GREATEST(virtual_now, start_tag of the job\n just dispatched): never decreases\n- System idle: highest finish_tag seen so far", "fontSize": 16.000000000000004, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "virtual_now (v5)\n- Universal tag, one row in scheduler_state\n- GREATEST(virtual_now, start_tag of the job\n just dispatched): never decreases\n- System idle: highest finish_tag seen so far", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Et7h3HJxnIeEuGY7-IS2k", "type": "text", "x": -11861.253199534589, "y": -5747.080625344379, "width": 143.66278076171875, "height": 14.064185871413207, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4v", "roundness": null, "seed": 297212562, "version": 212, "versionNonce": 2120970962, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "dispatch_key = start_tag", "fontSize": 11.251348697130565, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "dispatch_key = start_tag", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "kVHUVj7yIKmdhI0gEGunu", "type": "text", "x": -11856.4133194203, "y": -5828.948888620963, "width": 36.03208923339844, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4w", "roundness": null, "seed": 1960441938, "version": 111, "versionNonce": 836323474, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "DRR", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "DRR", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "8-EtoH3tAKc2TtjR3Zcj1", "type": "line", "x": -11858.109802653526, "y": -5810.749002210313, "width": 37.323563059789535, "height": 1.1310198864446193, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4x", "roundness": { "type": 2 }, "seed": 861609490, "version": 77, "versionNonce": 1820747346, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 37.323563059789535, -1.1310198864446193 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "YswT0r3yOY-72FWtjr1ud", "type": "line", "x": -11856.978751702787, "y": -5808.486962437424, "width": 36.758006520125946, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4y", "roundness": { "type": 2 }, "seed": 1513196498, "version": 84, "versionNonce": 390712338, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 36.758006520125946, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "g-0AuPnqyy892mrYE0Ac7", "type": "rectangle", "x": -10628.810376831761, "y": -4701.588874995091, "width": 722.756222786014, "height": 155.91176626304878, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b4z", "roundness": { "type": 3 }, "seed": 1064583570, "version": 383, "versionNonce": 1877435550, "isDeleted": false, "boundElements": [], "updated": 1794139726265, "created": 1794120896909, "link": null, "locked": false }, { "id": "9ZDdqqhJ0v97fGQB8Hup8", "type": "text", "x": -10599.120228959266, "y": -4686.320783018467, "width": 684.6992239643665, "height": 149.7855927776442, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b50", "roundness": null, "seed": 1582972754, "version": 428, "versionNonce": 1539898301, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794120896909, "link": null, "locked": false, "text": "## Caveats\n\n1. virtual_final_time = estimate i.e a guess, so a bad guess can cause \nunfairness in some rounds !!!\n2. (v5) Ties in start_tag: higher tier first, then lower enqueue_seq.\n No random choice, so replay tests are deterministic.", "fontSize": 19.97141237035256, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "## Caveats\n\n1. virtual_final_time = estimate i.e a guess, so a bad guess can cause \nunfairness in some rounds !!!\n2. (v5) Ties in start_tag: higher tier first, then lower enqueue_seq.\n No random choice, so replay tests are deterministic.", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "QM7GvFJUwk_NBL8rZU08N", "type": "text", "x": -13393.68858750633, "y": -6868.816610283248, "width": 800, "height": 350.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b51", "roundness": null, "seed": 859473170, "version": 843, "versionNonce": 2014636195, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794120896909, "link": null, "locked": false, "text": "Nomenclature: \n\n1. `tag` refers to start_tag/start_time which is a per tenant virtual time\nassigned to each job based on a universal virtual clock.\n2. `resource` refers to the containers the job runs on i.e in our case:\nsm(small),md(medium) and lg(large) running containers\n3. `execution slots` refers to capacity of worker to run 1 assigned(sm,md, or lg)\njob i.e sm 1..N, md 1..N, and lg 1..N, here N is the No. of execution slot\n4. `cap unit` refers to 1 unit of a tenant\u2019s concurrency cap\n5. `credit` refers to 1 free execution slot that the worker gives to the\ndispatcher\n6. `zombie worker` is a worker that the dispatcher thinks is dead, but which is\nstill alive and comes back later\n7. `fencing`: the store rejects a write whose attempt != current attempt", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Nomenclature: \n\n1. `tag` refers to start_tag/start_time which is a per tenant virtual time\nassigned to each job based on a universal virtual clock.\n2. `resource` refers to the containers the job runs on i.e in our case:\nsm(small),md(medium) and lg(large) running containers\n3. `execution slots` refers to capacity of worker to run 1 assigned(sm,md, or lg)\njob i.e sm 1..N, md 1..N, and lg 1..N, here N is the No. of execution slot\n4. `cap unit` refers to 1 unit of a tenant\u2019s concurrency cap\n5. `credit` refers to 1 free execution slot that the worker gives to the\ndispatcher\n6. `zombie worker` is a worker that the dispatcher thinks is dead, but which is\nstill alive and comes back later\n7. `fencing`: the store rejects a write whose attempt != current attempt", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "sqL99YfIAGJI0f-fW8iCI", "type": "diamond", "x": -11721.945436704553, "y": -6120.203281868318, "width": 129.9662780609107, "height": 127.75224304763769, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b52", "roundness": { "type": 2 }, "seed": 395055826, "version": 261, "versionNonce": 1801597714, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false }, { "id": "xY8Rg2yjHiLHmNFJ215ld", "type": "line", "x": -11721.945436704553, "y": -5928.244559496915, "width": 62.157726677946584, "height": 65.92487416963786, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b53", "roundness": { "type": 2 }, "seed": 501606546, "version": 98, "versionNonce": 177078482, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 62.157726677946584, -65.92487416963786 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "10aTi4iW__z1amopVMw8S", "type": "text", "x": -11691.808411972243, "y": -6081.070415559367, "width": 75.09991455078125, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b54", "roundness": null, "seed": 1845429842, "version": 88, "versionNonce": 1987712658, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "Tiered\nWeights\n\n", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Tiered\nWeights\n\n", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "5PtmP9E8Uh2yWHQpxPQjv", "type": "text", "x": -11859.926283418257, "y": -5898.601131572813, "width": 141.4722442626953, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b55", "roundness": null, "seed": 548301842, "version": 96, "versionNonce": 1071477842, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "( fair-share-core )", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "( fair-share-core )", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "8k54PDi46NeMibqidB-jI", "type": "text", "x": -11862.4038549197, "y": -5763.4196678780945, "width": 185.64375893600769, "height": 15.61267514814938, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b56", "roundness": null, "seed": 468392402, "version": 290, "versionNonce": 1217337874, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "(fair-share-dispatcher )", "fontSize": 12.490140118519504, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "(fair-share-dispatcher )", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "ZJEJgwq_-vRLLr1B6r6EJ", "type": "diamond", "x": -12044.64852411005, "y": -6100.928835616414, "width": 135.83080914412992, "height": 125.76929229035653, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b57", "roundness": { "type": 2 }, "seed": 1858867090, "version": 181, "versionNonce": 465729305, "isDeleted": false, "boundElements": [ { "id": "6M0bF0VGl-A-bcZCVs1t8", "type": "arrow" } ], "updated": 1794130704348, "created": 1794120896909, "link": null, "locked": false }, { "id": "mccZ8vOrYPP_poweshJsa", "type": "text", "x": -12007.420734888057, "y": -6064.614983595803, "width": 73.95991516113281, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b58", "roundness": null, "seed": 763907410, "version": 81, "versionNonce": 1557740946, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "leased\ntimeout", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "leased\ntimeout", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "nS_g0EyAPWQzMIFvOFnKf", "type": "line", "x": -11949.063782381016, "y": -6002.325732789733, "width": 92.56624245714534, "height": 75.46154221239527, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b59", "roundness": { "type": 2 }, "seed": 1299632914, "version": 85, "versionNonce": 2109717330, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 92.56624245714534, 75.46154221239527 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "5LqeIHU2zwf8nmwm57OqC", "type": "arrow", "x": -11421.907024951523, "y": -5933.017181447963, "width": 274.36616900762783, "height": 57.13440874834123, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5A", "roundness": { "type": 2 }, "seed": 17735890, "version": 151, "versionNonce": 959931166, "isDeleted": false, "boundElements": [], "updated": 1794138625289, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -274.36616900762783, 57.13440874834123 ] ], "startBinding": { "elementId": "eXI98ZEpQXChhpPqVEhmv", "mode": "orbit", "fixedPoint": [ 0.17557740659477472, 0.17557740659477458 ] }, "endBinding": { "elementId": "E2TpBxoT9anq7P_t9Q8d-", "mode": "orbit", "fixedPoint": [ 0.7786598171773841, 0.2213401828226172 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "gcGlApM_0FEhblkI-zH0Y", "type": "arrow", "x": -11418.61741179248, "y": -5683.333743144101, "width": 277.6557821666702, "height": 47.63737229269918, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5B", "roundness": { "type": 2 }, "seed": 333887122, "version": 201, "versionNonce": 2049804034, "isDeleted": false, "boundElements": [], "updated": 1794140974084, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -277.6557821666702, -47.63737229269918 ] ], "startBinding": { "elementId": "zxGbFiEPxx4mJADSt3kgs", "mode": "orbit", "fixedPoint": [ 0.1315922808569526, 0.8684077191430482 ] }, "endBinding": { "elementId": "E2TpBxoT9anq7P_t9Q8d-", "mode": "orbit", "fixedPoint": [ 0.675326422776683, 0.6753264227766823 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "AZr_AXqXCvzcFH-u9WvaZ", "type": "text", "x": -11674.102001867806, "y": -5699.609868703819, "width": 265.907693531464, "height": 48.99889281615565, "angle": 0.16114824583514764, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5D", "roundness": null, "seed": 726907410, "version": 633, "versionNonce": 1309797659, "isDeleted": false, "boundElements": [], "updated": 1794342265409, "created": 1794120896909, "link": null, "locked": false, "text": "application heartbeat\n(cpu_ms so far, lease )", "fontSize": 19.599557126462262, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "application heartbeat (cpu_ms so far, lease )", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "ujDbaIh4miofIjIHG7M9p", "type": "line", "x": -12890.142756939356, "y": -5761.554416946525, "width": 0, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5E", "roundness": { "type": 2 }, "seed": 11970514, "version": 50, "versionNonce": 86403090, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "0dnIP59K8mY1osA8v7ZE1", "type": "line", "x": -12320.94495721529, "y": -5805.218551288435, "width": 280.6529637973431, "height": 253.1977843777455, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5F", "roundness": { "type": 2 }, "seed": 250994066, "version": 122, "versionNonce": 671323602, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -280.6529637973431, 253.1977843777455 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "_s1uu1XHj3joa1SABmOpS", "type": "rectangle", "x": -12703.792218582592, "y": -5552.477782968571, "width": 248.48839345598145, "height": 244.99151551983394, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5G", "roundness": { "type": 3 }, "seed": 55073618, "version": 234, "versionNonce": 1530247774, "isDeleted": false, "boundElements": [ { "id": "Pf8_2s_TVA6fmrB0HGymc", "type": "arrow" } ], "updated": 1794140277244, "created": 1794120896909, "link": null, "locked": false }, { "id": "QtfDJWxQDj2oz0LqpkU44", "type": "text", "x": -12687.27009630965, "y": -5536.031661227497, "width": 226.6597900390625, "height": 225.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5H", "roundness": null, "seed": 1275404562, "version": 196, "versionNonce": 1059022249, "isDeleted": false, "boundElements": [ { "id": "eH_Pe1s5RbNMjTLxHoY-4", "type": "arrow" } ], "updated": 1794661218463, "created": 1794120896909, "link": null, "locked": false, "text": "Lifecycle (v5):\nQueued -> Assigned ->\nRunning{attempt,\nlease_expires_at,\ntimeout_at,\ncancel_requested}\n-> Completed | Failed\n| Cancelled | TimedOut\n(see v5 panel)", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Lifecycle (v5):\nQueued -> Assigned ->\nRunning{attempt,\nlease_expires_at,\ntimeout_at,\ncancel_requested}\n-> Completed | Failed\n| Cancelled | TimedOut\n(see v5 panel)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "pCi_eGr_nvYIvpW9aV1wa", "type": "arrow", "x": -12658.60814104032, "y": -5820.916377605447, "width": 337.2031543798592, "height": 6.365687527487978, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5I", "roundness": { "type": 2 }, "seed": 1956851410, "version": 163, "versionNonce": 1745932050, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 337.2031543798592, -6.365687527487978 ] ], "startBinding": { "elementId": "SNoDrfPBBafdS4lRnwNPx", "mode": "inside", "fixedPoint": [ 0.9999792068000627, 0.5168988555118098 ] }, "endBinding": { "elementId": "Myv5MZEGhzImarZ4DewhH", "mode": "inside", "fixedPoint": [ 0.002349390443736646, 0.5022204603754103 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "1xgJIQUDmc9NN6N3eRgJA", "type": "arrow", "x": -11866.372865003055, "y": -5918.317384094266, "width": 267.1812509383436, "height": 93.94889907212018, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5J", "roundness": { "type": 2 }, "seed": 28935314, "version": 115, "versionNonce": 314371806, "isDeleted": false, "boundElements": [], "updated": 1794138625288, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -267.1812509383436, -93.94889907212018 ] ], "startBinding": { "elementId": "E2TpBxoT9anq7P_t9Q8d-", "mode": "inside", "fixedPoint": [ 0.04408812259835801, 0.034676297915503855 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "BFPMzgDxifvzOOhbCSzua", "type": "rectangle", "x": -12239.553102039963, "y": -6068.963427074769, "width": 104.76646966509657, "height": 109.69663695766371, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5K", "roundness": { "type": 3 }, "seed": 1291788882, "version": 97, "versionNonce": 355578514, "isDeleted": false, "boundElements": [ { "id": "JdHRZvlEKC_nhsiDk6jVj", "type": "arrow" } ], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false }, { "id": "JdHRZvlEKC_nhsiDk6jVj", "type": "arrow", "x": -12231.21449352541, "y": -6059.273463422298, "width": 40.2214587982358, "height": 35.26773236075522, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5L", "roundness": { "type": 2 }, "seed": 1632307218, "version": 112, "versionNonce": 1564577874, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -40.2214587982358, -35.26773236075522 ] ], "startBinding": { "elementId": "BFPMzgDxifvzOOhbCSzua", "mode": "inside", "fixedPoint": [ 0.07959234038531302, 0.08833419073923757 ] }, "endBinding": { "elementId": "LcXtHh8wMu2EvjDLwMeHW", "mode": "inside", "fixedPoint": [ 0.995680941245345, 0.48432659741500406 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "zD0rg9-XiiomDodYVGyGp", "type": "text", "x": -12221.145455158809, "y": -6025.169218485368, "width": 56.959930419921875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5M", "roundness": null, "seed": 1452553682, "version": 52, "versionNonce": 1527756306, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896909, "link": null, "locked": false, "text": "Event", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Event", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Rxa9-fcGjqKrq6oHpw5DJ", "type": "line", "x": -12225.713201427696, "y": -6067.916215340992, "width": 4.815674257989485, "height": 199.84985344287873, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5N", "roundness": { "type": 2 }, "seed": 770812818, "version": 112, "versionNonce": 671661113, "isDeleted": false, "boundElements": [], "updated": 1794130752482, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -4.815674257989485, -199.84985344287873 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "IMkn8EBY8fopzt6NzBaFL", "type": "rectangle", "x": -12498.045865263952, "y": -6539.983946009495, "width": 375.62140172882914, "height": 270.88086688242043, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5O", "roundness": { "type": 3 }, "seed": 765094226, "version": 136, "versionNonce": 673243577, "isDeleted": false, "boundElements": [ { "id": "rNUE_3ZYM7f28m72UAYx9", "type": "arrow" } ], "updated": 1794131217289, "created": 1794120896909, "link": null, "locked": false }, { "id": "ZKFo63l4acsVul4CsNwIi", "type": "text", "x": -12488.670645820966, "y": -6494.173958752097, "width": 361.10239554740286, "height": 175.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5P", "roundness": null, "seed": 1021669138, "version": 180, "versionNonce": 1635342799, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794120896909, "link": null, "locked": false, "text": "1. Dispatch { attempt, ack_deadline },\n2. Complete { attempt: u32 },\n3. Fail { attempt: u32 },\n4. Cancel {reason}\n( Ensured Idempotency ),\n5. LeaseExpired { attempt: u32 },\n6. Timedout { attempt: u32 }", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "1. Dispatch { attempt, ack_deadline },\n2. Complete { attempt: u32 },\n3. Fail { attempt: u32 },\n4. Cancel {reason}\n( Ensured Idempotency ),\n5. LeaseExpired { attempt: u32 },\n6. Timedout { attempt: u32 }", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Pf8_2s_TVA6fmrB0HGymc", "type": "arrow", "x": -12703.155872785896, "y": -5436.885544088584, "width": 128.91782275319565, "height": 154.6604156884723, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5Q", "roundness": { "type": 2 }, "seed": 1078574290, "version": 151, "versionNonce": 529186462, "isDeleted": false, "boundElements": [], "updated": 1794140277245, "created": 1794120896909, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -128.91782275319565, -154.6604156884723 ] ], "startBinding": { "elementId": "_s1uu1XHj3joa1SABmOpS", "mode": "inside", "fixedPoint": [ 0.0025608672817507305, 0.4718213960786289 ] }, "endBinding": { "elementId": "SNoDrfPBBafdS4lRnwNPx", "mode": "inside", "fixedPoint": [ 0.595012633661607, 0.9980351043720818 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "_EiHDTTW87rV1vSw4szue", "type": "text", "x": -12842.979075484676, "y": -5522.861132281, "width": 75.89991760253906, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5R", "roundness": null, "seed": 1969115794, "version": 93, "versionNonce": 1667345106, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896910, "link": null, "locked": false, "text": "Status \nupdate", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Status \nupdate", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "n61QXt9hJAc-8gd3XVsBJ", "type": "text", "x": -11858.750823570403, "y": -6057.486216666304, "width": 108.4696756176575, "height": 59.34360098909651, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5S", "roundness": null, "seed": 1322922066, "version": 743, "versionNonce": 189300882, "isDeleted": false, "boundElements": [], "updated": 1794120899055, "created": 1794120896910, "link": null, "locked": false, "text": "- cpu seconds not wall\nseconds\n- Different resource,\ndifferent speed, one\nconsistent cost unit", "fontSize": 9.494976158255442, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "- cpu seconds not wall seconds\n- Different resource, different speed, one consistent cost unit", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "NMjAyXHq1PjT90mDOuafv", "type": "line", "x": -12046.76101818078, "y": -5117.9240232601505, "width": 160.65039201814022, "height": 1.889993212018453, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5T", "roundness": { "type": 2 }, "seed": 1351898642, "version": 196, "versionNonce": 1908486755, "isDeleted": false, "boundElements": [], "updated": 1794621580572, "created": 1794120896910, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 160.65039201814022, -1.889993212018453 ], [ 0, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "v0GvQuxSlFj2VNH67laA8", "type": "line", "x": -12046.131031979112, "y": -5122.355242934791, "width": 161.9104336355132, "height": 2.5200140207048207, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5U", "roundness": { "type": 2 }, "seed": 1803797458, "version": 176, "versionNonce": 528821635, "isDeleted": false, "boundElements": [], "updated": 1794621577571, "created": 1794120896910, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 161.9104336355132, 2.5200140207048207 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "UUoHjyyWalMBSzJDaTIlF", "type": "line", "x": -12044.871024968761, "y": -5117.309935758923, "width": 157.5004264027899, "height": 0.6299862016660427, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5V", "roundness": { "type": 2 }, "seed": 1505720722, "version": 187, "versionNonce": 284467203, "isDeleted": false, "boundElements": [], "updated": 1794621573683, "created": 1794120896910, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 157.5004264027899, -0.6299862016660427 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "6M0bF0VGl-A-bcZCVs1t8", "type": "arrow", "x": -11976.578315555453, "y": -6102.491133211775, "width": 0.3598997900589893, "height": 28.20137241814882, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5W", "roundness": { "type": 2 }, "seed": 1310371961, "version": 38, "versionNonce": 107061465, "isDeleted": false, "boundElements": [], "updated": 1794130707261, "created": 1794130704342, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0.3598997900589893, -28.20137241814882 ] ], "startBinding": { "elementId": "ZJEJgwq_-vRLLr1B6r6EJ", "mode": "orbit", "fixedPoint": [ 0.5006228000000006, 0.031307324135288236 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "Ixx4CAeY-HfuVPOI4R3sl", "type": "rectangle", "x": -12206.445557078549, "y": -6169.242122863179, "width": 352.3009831387153, "height": 36.407993072719364, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5X", "roundness": { "type": 3 }, "seed": 1218941815, "version": 163, "versionNonce": 39289271, "isDeleted": false, "boundElements": [], "updated": 1794130767858, "created": 1794130712174, "link": null, "locked": false }, { "id": "lSBnBiAoya68jeWISNsL5", "type": "text", "x": -12201.350319991492, "y": -6163.373799799386, "width": 338.8743968316472, "height": 22.858346428417477, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5a", "roundness": null, "seed": 1535688407, "version": 165, "versionNonce": 325459991, "isDeleted": false, "boundElements": [], "updated": 1794130983032, "created": 1794130734770, "link": null, "locked": false, "text": "The worker is dead or unreachable", "fontSize": 18.286677142733982, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "The worker is dead or unreachable", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "rNUE_3ZYM7f28m72UAYx9", "type": "arrow", "x": -12334.233480803405, "y": -6545.483946009495, "width": 1.6590297357379313, "height": 78.96276906379353, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5d", "roundness": { "type": 2 }, "seed": 1858803481, "version": 50, "versionNonce": 2094798329, "isDeleted": false, "boundElements": [], "updated": 1794131218811, "created": 1794131217287, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1.6590297357379313, -78.96276906379353 ] ], "startBinding": { "elementId": "IMkn8EBY8fopzt6NzBaFL", "mode": "orbit", "fixedPoint": [ 0.42929814498924185, 0.429298144989241 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "33cBfBnTdYUJhikAGYMV9", "type": "text", "x": -12392.116703360312, "y": -6712.938656308644, "width": 800, "height": 75, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5e", "roundness": null, "seed": 220095703, "version": 8, "versionNonce": 2044574295, "isDeleted": false, "boundElements": [], "updated": 1794131331778, "created": 1794131222410, "link": null, "locked": false, "text": "Why each even carries attempt ?\n- to prevent stale workers from completing or modifying a job after its lease\nexpires and a newer attempt takes over", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Why each even carries attempt ?\n- to prevent stale workers from completing or modifying a job after its lease expires and a newer attempt takes over", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "208ntaicegev3vvjVPmdV", "type": "text", "x": -8281.49755777791, "y": -6402.08854671562, "width": 105.159912109375, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5q", "roundness": null, "seed": 271452345, "version": 11, "versionNonce": 1225467257, "isDeleted": false, "boundElements": [], "updated": 1794132709007, "created": 1794132701734, "link": null, "locked": false, "text": "GRPC flow", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "GRPC flow", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "zgjSntUVA6m3WABVHsRTC", "type": "text", "x": -8355.884423363685, "y": -6344.425462992086, "width": 702.498779296875, "height": 75, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5s", "roundness": null, "seed": 152266073, "version": 10, "versionNonce": 1734265145, "isDeleted": false, "boundElements": [], "updated": 1794132752351, "created": 1794132735895, "link": null, "locked": false, "text": "Client \u2500\u2500Cancel(job)\u2500\u2500> Dispatcher \u2500\u2500Cancel{job, attempt}\u2500\u2500> Worker\n \u2502\n \u2514\u2500\u2500\u2500\u2500 Finished{Cancelled, cpu} \u2500\u2500\u2500\u2500\u2518", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Client \u2500\u2500Cancel(job)\u2500\u2500> Dispatcher \u2500\u2500Cancel{job, attempt}\u2500\u2500> Worker\n \u2502\n \u2514\u2500\u2500\u2500\u2500 Finished{Cancelled, cpu} \u2500\u2500\u2500\u2500\u2518", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "2ir3Q78DfFrIvi_-6DgcQ", "type": "text", "x": -16449.547028828703, "y": -6612.453831983352, "width": 351.83244154137435, "height": 76.0080856134891, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5t", "roundness": null, "seed": 957785401, "version": 182, "versionNonce": 1534466189, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133003514, "link": null, "locked": false, "text": "State Machines in Rust\n", "fontSize": 30.40323424539564, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "State Machines in Rust\n", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "szrpr6w5Lqt5cHf1eZXkb", "type": "rectangle", "x": -16608.15390828628, "y": -6386.102042338903, "width": 248.25921451873182, "height": 209.2470698636389, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5u", "roundness": { "type": 3 }, "seed": 1540686999, "version": 45, "versionNonce": 1358906051, "isDeleted": true, "boundElements": [ { "id": "dHrOnuqhaEtt6owV_mMH4", "type": "arrow" } ], "updated": 1794636656461, "created": 1794133026280, "link": null, "locked": false }, { "id": "GMLJYw2O9F-apZLnRcQjN", "type": "text", "x": -16555.664912955905, "y": -6154.244097484209, "width": 141.5029296875, "height": 35.63967876681909, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5v", "roundness": null, "seed": 1483216503, "version": 85, "versionNonce": 865959661, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133028605, "link": null, "locked": false, "text": "Job State", "fontSize": 28.51174301345527, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Job State", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "dHrOnuqhaEtt6owV_mMH4", "type": "arrow", "x": -16354.394693767548, "y": -6281.470643980168, "width": 252.21660573898953, "height": 0.5989584956123508, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5x", "roundness": { "type": 2 }, "seed": 2116534745, "version": 55, "versionNonce": 2117071459, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133057804, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 252.21660573898953, -0.5989584956123508 ] ], "startBinding": { "elementId": "szrpr6w5Lqt5cHf1eZXkb", "mode": "orbit", "fixedPoint": [ 1, 0.5001 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "3v7VaT7g24TlbAtKxWmsd", "type": "rectangle", "x": -16100.995800481996, "y": -6399.106068910785, "width": 243.53045396915877, "height": 215.1579231414371, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5y", "roundness": { "type": 3 }, "seed": 1542270487, "version": 69, "versionNonce": 852040013, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133061847, "link": null, "locked": false }, { "id": "rmEP_Zy54hrBn8ryL7FUb", "type": "text", "x": -16183.512535699228, "y": -6167.248156525813, "width": 457.22174072265625, "height": 36.82183643449024, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b5z", "roundness": null, "seed": 1989784249, "version": 103, "versionNonce": 216641027, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133068905, "link": null, "locked": false, "text": "Transition (Changed,Unchanged)", "fontSize": 29.457469147592192, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Transition (Changed,Unchanged)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "SilCZYrzStmiVdRYdGeaU", "type": "text", "x": -16304.68655397108, "y": -6324.124290971172, "width": 132.35581751546766, "height": 33.27533096175466, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b60", "roundness": null, "seed": 1872084951, "version": 23, "versionNonce": 1381132205, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133078195, "link": null, "locked": false, "text": "Job Event", "fontSize": 26.62026476940372, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Job Event", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "fvdAx1OO6BholAoDA8HVR", "type": "ellipse", "x": -16209.032551848493, "y": -5965.489682305427, "width": 226.22333243637695, "height": 207.70214008202674, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b62", "roundness": { "type": 2 }, "seed": 219471639, "version": 45, "versionNonce": 2815395, "isDeleted": true, "boundElements": [ { "id": "-ZlgtaZ9lkGS__25e26VR", "type": "arrow" } ], "updated": 1794636656461, "created": 1794133367108, "link": null, "locked": false }, { "id": "QpNJP_mA8lT6BsVnHuc_3", "type": "ellipse", "x": -15722.189324599269, "y": -5786.901843180453, "width": 226.22333243637877, "height": 218.28568376104522, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b63", "roundness": { "type": 2 }, "seed": 999932151, "version": 68, "versionNonce": 610462221, "isDeleted": true, "boundElements": [ { "id": "81n5ZA2DAdIvwyEcn2B4S", "type": "arrow" } ], "updated": 1794636656461, "created": 1794133369342, "link": null, "locked": false }, { "id": "Jc5nvF1lEzH4TgGEoZ4Kv", "type": "ellipse", "x": -15858.452517596123, "y": -5403.238701801228, "width": 263.2657171450792, "height": 265.9116848202211, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b64", "roundness": { "type": 2 }, "seed": 1088122391, "version": 37, "versionNonce": 1998735683, "isDeleted": true, "boundElements": [ { "id": "a523-PF5g4uIhQ5Sx5Q6K", "type": "arrow" } ], "updated": 1794636656461, "created": 1794133372262, "link": null, "locked": false }, { "id": "NapfrIfBHKhOfGoFUr-dH", "type": "ellipse", "x": -16714.86451608406, "y": -5843.467324526211, "width": 226.22333243637695, "height": 210.3480714214411, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b65", "roundness": { "type": 2 }, "seed": 2022056823, "version": 96, "versionNonce": 972300397, "isDeleted": true, "boundElements": [ { "id": "pyoIJkKQW8UNHVHrI40s_", "type": "arrow" } ], "updated": 1794636656461, "created": 1794133374510, "link": null, "locked": false }, { "id": "8EItOvA0Rgdhj2dSIln-G", "type": "ellipse", "x": -16403.22938206741, "y": -5396.394065796166, "width": 224.9004212702621, "height": 218.28564742531762, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b67", "roundness": { "type": 2 }, "seed": 1323273527, "version": 74, "versionNonce": 657336547, "isDeleted": true, "boundElements": [ { "id": "pyoIJkKQW8UNHVHrI40s_", "type": "arrow" }, { "id": "-ZlgtaZ9lkGS__25e26VR", "type": "arrow" } ], "updated": 1794636656461, "created": 1794133379481, "link": null, "locked": false }, { "id": "ngA-oUx0FZPxLuFw9olVm", "type": "text", "x": -16684.644307281254, "y": -5755.347466925216, "width": 102.85993957519531, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6G", "roundness": null, "seed": 854489975, "version": 58, "versionNonce": 1221561037, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133444318, "link": null, "locked": false, "text": " Queued", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": " Queued", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Rd0gHlj8JqMPKdmPuIy2E", "type": "text", "x": -16177.288552483767, "y": -5872.665600980705, "width": 73.21992492675781, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6I", "roundness": null, "seed": 1041351833, "version": 10, "versionNonce": 366810243, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133451452, "link": null, "locked": false, "text": "Running", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Running", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "vbkkF0adxCYD48peRomcW", "type": "text", "x": -15688.5367177552, "y": -5701.602458825707, "width": 97.71990966796875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6J", "roundness": null, "seed": 684127801, "version": 12, "versionNonce": 294562093, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133456019, "link": null, "locked": false, "text": "Completed", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Completed", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "NfN6fHynGJBumE2-TOGUg", "type": "text", "x": -15786.287084700914, "y": -5292.272797240532, "width": 63.95994567871094, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6K", "roundness": null, "seed": 696912281, "version": 10, "versionNonce": 317557795, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133459948, "link": null, "locked": false, "text": "Failed ", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Failed ", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "mwpfGyyAFDfps2rupbGdg", "type": "text", "x": -16346.824345155239, "y": -5304.491593108745, "width": 87.27992248535156, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6L", "roundness": null, "seed": 513039161, "version": 22, "versionNonce": 94043021, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133463308, "link": null, "locked": false, "text": "Cancelled", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Cancelled", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "DXacatwB_IGbWI9i7USYE", "type": "text", "x": -15470.347524701281, "y": -6162.553865826961, "width": 351.5997009277344, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6M", "roundness": null, "seed": 924150967, "version": 3, "versionNonce": 1917718467, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133535882, "link": null, "locked": false, "text": "1. Dispatch { lease_until: Instant },", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "1. Dispatch { lease_until: Instant },", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "QrJj7XFTAKtjSqRJek2Vb", "type": "text", "x": -15441.067556439562, "y": -6127.553865826961, "width": 293.0397644042969, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6N", "roundness": null, "seed": 13508153, "version": 3, "versionNonce": 757989869, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133535882, "link": null, "locked": false, "text": "2. Complete { attempt: u32 },", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "2. Complete { attempt: u32 },", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "h9UIM5cZys-TCYdLDGYch", "type": "text", "x": -15413.897573529406, "y": -6092.553865826961, "width": 238.69979858398438, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6O", "roundness": null, "seed": 1364508119, "version": 3, "versionNonce": 102571875, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133535882, "link": null, "locked": false, "text": "3. Fail { attempt: u32 },", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "3. Fail { attempt: u32 },", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "EFv83k9DYPUSRMhFZRD6P", "type": "text", "x": -15464.057531415148, "y": -6057.553865826961, "width": 339.01971435546875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6P", "roundness": null, "seed": 700444441, "version": 3, "versionNonce": 1297169485, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133535882, "link": null, "locked": false, "text": "4. Cancel ( Ensured Idempotency ),", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "4. Cancel ( Ensured Idempotency ),", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "8-xCWErZZpKUJyiUPZbjT", "type": "text", "x": -15460.927526532336, "y": -6022.553865826961, "width": 332.75970458984375, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6Q", "roundness": null, "seed": 1171057399, "version": 3, "versionNonce": 1487485699, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133535882, "link": null, "locked": false, "text": "5. LeaseExpired { attempt: u32 },", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "5. LeaseExpired { attempt: u32 },", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "wPnvwEx2OsbWwCxDtuKZj", "type": "text", "x": -15441.45755582921, "y": -5987.553865826961, "width": 293.81976318359375, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6R", "roundness": null, "seed": 2047124473, "version": 3, "versionNonce": 1306405549, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133535882, "link": null, "locked": false, "text": "6. Timedout { attempt: u32 }", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "6. Timedout { attempt: u32 }", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "pyoIJkKQW8UNHVHrI40s_", "type": "arrow", "x": -16612.753280038567, "y": -5635.953599107622, "width": 205.09354527826872, "height": 333.29791544641466, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6S", "roundness": { "type": 2 }, "seed": 1757187159, "version": 93, "versionNonce": 644932259, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133552905, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 205.09354527826872, 333.29791544641466 ] ], "startBinding": { "elementId": "NapfrIfBHKhOfGoFUr-dH", "mode": "inside", "fixedPoint": [ 0.45137358267061745, 0.9865254481122716 ] }, "endBinding": { "elementId": "8EItOvA0Rgdhj2dSIln-G", "mode": "orbit", "fixedPoint": [ 0.022508367003266652, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "-ZlgtaZ9lkGS__25e26VR", "type": "arrow", "x": -16105.689545182973, "y": -5752.659031686782, "width": 134.33755109306185, "height": 361.9204437655544, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6T", "roundness": { "type": 2 }, "seed": 2116500409, "version": 44, "versionNonce": 1602112781, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133589877, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -134.33755109306185, 361.9204437655544 ] ], "startBinding": { "elementId": "fvdAx1OO6BholAoDA8HVR", "mode": "orbit", "fixedPoint": [ 0.5001, 0.8976885743537707 ] }, "endBinding": { "elementId": "8EItOvA0Rgdhj2dSIln-G", "mode": "orbit", "fixedPoint": [ 0.5548312110305924, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "a523-PF5g4uIhQ5Sx5Q6K", "type": "arrow", "x": -15616.198073180587, "y": -5196.507087776704, "width": 213.19558880889417, "height": 172.68016548085143, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6V", "roundness": { "type": 2 }, "seed": 597156281, "version": 231, "versionNonce": 404919875, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133675192, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 188.48339821910668, -120.71378096586432 ], [ -24.712190589787497, -172.68016548085143 ] ], "startBinding": { "elementId": "Jc5nvF1lEzH4TgGEoZ4Kv", "mode": "inside", "fixedPoint": [ 0.9201898638478445, 0.7774446398032189 ] }, "endBinding": { "elementId": "Jc5nvF1lEzH4TgGEoZ4Kv", "mode": "inside", "fixedPoint": [ 0.8263219996315203, 0.12805548040017165 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "O_9XYD6eqW0teyAReVagc", "type": "text", "x": -15413.323977108572, "y": -5342.5126245656675, "width": 173.27981567382812, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6W", "roundness": null, "seed": 251836761, "version": 27, "versionNonce": 1638605677, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133701993, "link": null, "locked": false, "text": "Cancellation Error", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Cancellation Error", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "81n5ZA2DAdIvwyEcn2B4S", "type": "arrow", "x": -15504.309146769985, "y": -5626.6249769571095, "width": 225.18783701965003, "height": 155.35804950833426, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6X", "roundness": { "type": 2 }, "seed": 202470105, "version": 289, "versionNonce": 1016911331, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133712723, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 188.48339821910668, -120.71378096586432 ], [ -36.70443880054336, -155.35804950833426 ] ], "startBinding": null, "endBinding": { "elementId": "QpNJP_mA8lT6BsVnHuc_3", "mode": "orbit", "fixedPoint": [ 0.800871143915682, 0.022533849358596366 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "SrZhh8rs6pmo110-YaeBv", "type": "text", "x": -15307.792192853914, "y": -5773.034335331828, "width": 173.27981567382812, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6Y", "roundness": null, "seed": 152751191, "version": 25, "versionNonce": 145632717, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133723141, "link": null, "locked": false, "text": "Cancellation Error", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Cancellation Error", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "--Kgf9hn_1eHwROr4FPhv", "type": "text", "x": -16150.847242070102, "y": -5584.75603842295, "width": 87.27992248535156, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6Z", "roundness": null, "seed": 739694809, "version": 12, "versionNonce": 1122902403, "isDeleted": true, "boundElements": [], "updated": 1794636656461, "created": 1794133729406, "link": null, "locked": false, "text": "Cancelled", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Cancelled", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Y2nAiqrsC9gzDnOzVV3EC", "type": "text", "x": -11288.942758842804, "y": -5915.0190414062845, "width": 49.279937744140625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6a", "roundness": null, "seed": 426180098, "version": 46, "versionNonce": 938596766, "isDeleted": false, "boundElements": [], "updated": 1794138646541, "created": 1794138613576, "link": null, "locked": false, "text": "slots", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "slots", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "8ZKQxCydn07rip-rp4R_h", "type": "text", "x": -11285.705220705098, "y": -5816.085952141107, "width": 49.279937744140625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6b", "roundness": null, "seed": 933086082, "version": 46, "versionNonce": 574840706, "isDeleted": false, "boundElements": [], "updated": 1794138642402, "created": 1794138621912, "link": null, "locked": false, "text": "slots", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "slots", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "ptvwWUjXTutCEexkQZZOo", "type": "text", "x": -11278.277803044954, "y": -5711.1118759310275, "width": 49.279937744140625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6c", "roundness": null, "seed": 749333086, "version": 59, "versionNonce": 1903923998, "isDeleted": false, "boundElements": [], "updated": 1794138655159, "created": 1794138623607, "link": null, "locked": false, "text": "slots", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "slots", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "eH_Pe1s5RbNMjTLxHoY-4", "type": "arrow", "x": -12535.256762987747, "y": -5387.438684706168, "width": 1.5186166868006694, "height": 120.7296720663935, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6e", "roundness": { "type": 2 }, "seed": 1832877982, "version": 68, "versionNonce": 1043401822, "isDeleted": false, "boundElements": [], "updated": 1794140302902, "created": 1794140300553, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1.5186166868006694, 120.7296720663935 ] ], "startBinding": { "elementId": "QtfDJWxQDj2oz0LqpkU44", "mode": "inside", "fixedPoint": [ 0.6706674055230782, 0.6604132289836869 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "W5IQF8X2ny8-7MEpZhuIu", "type": "rectangle", "x": -12706.89143154447, "y": -5260.634567664762, "width": 199.39553975456366, "height": 48.21590253878103, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6f", "roundness": { "type": 3 }, "seed": 582367234, "version": 378, "versionNonce": 2101010014, "isDeleted": false, "boundElements": [], "updated": 1794140388144, "created": 1794140307328, "link": null, "locked": false }, { "id": "HfcYcfQusEFc8WzDGwwML", "type": "text", "x": -12701.535604065872, "y": -5257.341020857903, "width": 185.64417067224838, "height": 40.31884469034091, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6g", "roundness": null, "seed": 266461314, "version": 184, "versionNonce": 372669662, "isDeleted": false, "boundElements": [], "updated": 1794140365635, "created": 1794140313022, "link": null, "locked": false, "text": "better than a separate\nCancelling state", "fontSize": 16.127537876136365, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "better than a separate \nCancelling state", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "60ooWmsQobqfNT3OALfa8", "type": "text", "x": -12483.455383678629, "y": -6531.271271750756, "width": 157.1598358154297, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6j", "roundness": null, "seed": 842151326, "version": 19, "versionNonce": 1989995038, "isDeleted": false, "boundElements": [], "updated": 1794140488669, "created": 1794140483994, "link": null, "locked": false, "text": "Events lifecycle:", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Events lifecycle:", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "4Y0S8945RQ2yrKuUlJZHy", "type": "line", "x": -12487.55211611929, "y": -6509.348746629896, "width": 167.14710665561506, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6k", "roundness": { "type": 2 }, "seed": 119375426, "version": 61, "versionNonce": 59860866, "isDeleted": false, "boundElements": [], "updated": 1794140497819, "created": 1794140496110, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 167.14710665561506, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "j0tEZ4egfCAJgEVReLRUP", "type": "line", "x": -12488.37144460416, "y": -6501.974610233444, "width": 169.60513711837848, "height": 2.458052966843752, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6l", "roundness": { "type": 2 }, "seed": 1724971550, "version": 73, "versionNonce": 257505246, "isDeleted": false, "boundElements": [], "updated": 1794140505321, "created": 1794140503006, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 169.60513711837848, -2.458052966843752 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "fXbkeqzpnXLlYENCHUnz_", "type": "line", "x": -12057.394129654194, "y": -6506.890698559773, "width": 108.97338897703048, "height": 2.4580529668428426, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6m", "roundness": { "type": 2 }, "seed": 505978014, "version": 45, "versionNonce": 386958174, "isDeleted": false, "boundElements": [], "updated": 1794140514616, "created": 1794140513798, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 108.97338897703048, -2.4580529668428426 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "NBiYqKsanRBKQNrLF2si6", "type": "line", "x": -12077.877791857496, "y": -6351.214483245349, "width": 90.1282937271535, "height": 1.6387019778949252, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6o", "roundness": { "type": 2 }, "seed": 586691202, "version": 45, "versionNonce": 945782722, "isDeleted": false, "boundElements": [], "updated": 1794140530492, "created": 1794140528789, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 90.1282937271535, -1.6387019778949252 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "p09HgM1Jnkr8-m8E-RprJ", "type": "line", "x": -11772.260795647204, "y": -6356.949917663905, "width": 153.2181173394256, "height": 1.6387019778958347, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6p", "roundness": { "type": 2 }, "seed": 182425054, "version": 55, "versionNonce": 257767198, "isDeleted": false, "boundElements": [], "updated": 1794140535718, "created": 1794140534574, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 153.2181173394256, 1.6387019778958347 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "ArdZBsg9fVKhUyj9okmyL", "type": "text", "x": -11699.023933859284, "y": -5675.144665591083, "width": 268.7691955566406, "height": 38.85617191201355, "angle": 0.12009154786316412, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6t", "roundness": null, "seed": 1350993118, "version": 235, "versionNonce": 1154958005, "isDeleted": false, "boundElements": [], "updated": 1794342255085, "created": 1794140927149, "link": null, "locked": false, "text": " \nshows that the job process is alive", "fontSize": 15.54246876480542, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": " \nshows that the job process is alive", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "RQHzsi8PYZkAhIIshRi-2", "type": "text", "x": -11608.307046389176, "y": -5927.8106266864825, "width": 102.94418334960938, "height": 20, "angle": 6.093281464502996, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6v", "roundness": null, "seed": 964094658, "version": 220, "versionNonce": 406536450, "isDeleted": false, "boundElements": [], "updated": 1794141147093, "created": 1794141103305, "link": null, "locked": false, "text": "PING / lease", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "PING / lease", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "q-4JBqet_qee9mTgAIqPf", "type": "text", "x": -11698.862942343476, "y": -5948.696082859071, "width": 316.7685852050781, "height": 20, "angle": 6.0770454463357195, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6w", "roundness": null, "seed": 226150686, "version": 257, "versionNonce": 1805499166, "isDeleted": false, "boundElements": [], "updated": 1794141502589, "created": 1794141279271, "link": null, "locked": false, "text": "HTTP Keepalive / Persistent Connection", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "HTTP Keepalive / Persistent Connection", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "5tnqShp7hNcWtHffn9txT", "type": "text", "x": -11672.742608395662, "y": -5961.597199304203, "width": 238.92848205566406, "height": 20, "angle": 6.087487786832691, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6x", "roundness": null, "seed": 1368926338, "version": 179, "versionNonce": 667994562, "isDeleted": false, "boundElements": [], "updated": 1794141506565, "created": 1794141462115, "link": null, "locked": false, "text": "shows that the worker is alive", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "shows that the worker is alive", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "TAXEejJJ9ALrKNMH7p4ld", "type": "line", "x": -11478.787474130157, "y": -5980.159943524175, "width": 1.610376336322588, "height": 158.62596148985813, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6y", "roundness": { "type": 2 }, "seed": 205637250, "version": 51, "versionNonce": 243292738, "isDeleted": false, "boundElements": [], "updated": 1794141686503, "created": 1794141685533, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1.610376336322588, -158.62596148985813 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "3CYrK5zouP0Isvp3gNbcB", "type": "rectangle", "x": -11503.748926582579, "y": -7272.285818074961, "width": 839.9033873384709, "height": 1130.2790719255077, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b6z", "roundness": { "type": 3 }, "seed": 1693823234, "version": 525, "versionNonce": 35775789, "isDeleted": false, "boundElements": [ { "id": "-H4f-p1s-k0rZQgZIVikv", "type": "arrow" } ], "updated": 1794636739297, "created": 1794141689964, "link": null, "locked": false }, { "id": "wiLhaD5uuktIg4PIWOafp", "type": "text", "x": -11489.05662063127, "y": -7246.517090373808, "width": 811.7916268288328, "height": 625.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b70", "roundness": null, "seed": 260716354, "version": 1809, "versionNonce": 437976712, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794141697594, "link": null, "locked": false, "text": "## Lease & Hearbeat\nUse both. Keepalive finds a dead connection fast. A heartbeat shows that the\njob is healthy. A worker can have a live connection while its job process hangs.\n\n***Self-fencing***: Worker need to kill itself after lease_timeout or\nheartbeat_timeout as the dispatcher cannot reach it !!! Why:\n1. The worker process freezes\n2. Messages inflight, the worker sent after loosing connection\n\nA) if lease/ping does not arrive: This is a warning not a verdict that the worker\ndied so wait till lease_timeout and then requeue with attempt. Dispatcher cannot\nreach the worker. (v5) Worker KILLS its container after self_kill_after (25 s),\nBEFORE the requeue at lease_timeout (30 s). DB fencing stays as a 2nd guard, e.g.:\nW1 sends Finished{J, attempt=1, Failed} t=90s\nW2 sends: Finished{J, attempt=2, Completed}.\nThen through database only accept the latest attempt only !\n```sql\nUPDATE jobs\nSET state = 'failed', ...\nWHERE id = 'J'\n AND attempt = 1 -- the attempt in W1's message\n AND state = 'running';\n```\nThe database rejects the stale write itself.\nB) if heartbeat does not arrive: ", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "## Lease & Hearbeat\nUse both. Keepalive finds a dead connection fast. A heartbeat shows that the\njob is healthy. A worker can have a live connection while its job process hangs.\n\n***Self-fencing***: Worker need to kill itself after lease_timeout or\nheartbeat_timeout as the dispatcher cannot reach it !!! Why:\n1. The worker process freezes\n2. Messages inflight, the worker sent after loosing connection\n\nA) if lease/ping does not arrive: This is a warning not a verdict that the worker\ndied so wait till lease_timeout and then requeue with attempt. Dispatcher cannot\nreach the worker. (v5) Worker KILLS its container after self_kill_after (25 s),\nBEFORE the requeue at lease_timeout (30 s). DB fencing stays as a 2nd guard, e.g.:\nW1 sends Finished{J, attempt=1, Failed} t=90s\nW2 sends: Finished{J, attempt=2, Completed}.\nThen through database only accept the latest attempt only !\n```sql\nUPDATE jobs\nSET state = 'failed', ...\nWHERE id = 'J'\n AND attempt = 1 -- the attempt in W1's message\n AND state = 'running';\n```\nThe database rejects the stale write itself.\nB) if heartbeat does not arrive: ", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "yhMvd7iAIm99Kcdn5s1-q", "type": "text", "x": -11437.302158963137, "y": -6544.37249041442, "width": 662.9794311523438, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b71", "roundness": null, "seed": 1406343489, "version": 290, "versionNonce": 1962094863, "isDeleted": false, "boundElements": [], "updated": 1794255198108, "created": 1794255148592, "link": null, "locked": false, "text": "heartbeat_interval 5 s Low cost, fast enough detection", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "heartbeat_interval 5 s Low cost, fast enough detection", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "FUvZsZzzaE9navAB8KGm7", "type": "text", "x": -11501.952122342043, "y": -6509.37249041442, "width": 792.2793579101562, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b72", "roundness": null, "seed": 1500960591, "version": 229, "versionNonce": 1420310319, "isDeleted": false, "boundElements": [], "updated": 1794255198108, "created": 1794255148592, "link": null, "locked": false, "text": "lease_timeout 30 s (6 heartbeats) Tolerates short network problems", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "lease_timeout 30 s (6 heartbeats) Tolerates short network problems", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "dSGLJfSPn-3MkFawFWhXS", "type": "text", "x": -11420.172184597903, "y": -6474.37249041442, "width": 628.719482421875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b73", "roundness": null, "seed": 1065371937, "version": 259, "versionNonce": 538409295, "isDeleted": false, "boundElements": [], "updated": 1794255198108, "created": 1794255148592, "link": null, "locked": false, "text": "worker_self_kill_after 25 s Less than lease_timeout", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "worker_self_kill_after 25 s Less than lease_timeout", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "UcCdcQlz8t9pkQKmGTT-o", "type": "text", "x": -11479.77216018384, "y": -6439.37249041442, "width": 747.91943359375, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b74", "roundness": null, "seed": 777852271, "version": 229, "versionNonce": 1553561455, "isDeleted": false, "boundElements": [], "updated": 1794255198108, "created": 1794255148592, "link": null, "locked": false, "text": "gRPC keepalive 10 s interval, 5 s timeout Finds dead TCP quickly", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "gRPC keepalive 10 s interval, 5 s timeout Finds dead TCP quickly", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "_AkzDK2j-D8XTAf663jTp", "type": "rectangle", "x": -9132.90108588686, "y": -8609.777384005776, "width": 231.11108398437503, "height": 207.77780151367188, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7B", "roundness": { "type": 3 }, "seed": 1574956251, "version": 121, "versionNonce": 968912507, "isDeleted": false, "boundElements": [ { "id": "miOBTiXeIB2zMFpZQTLIk", "type": "arrow" }, { "id": "eo21HiWyZKO513s0EUTwI", "type": "arrow" }, { "id": "2ivtZ93NGBDK050ZF6QI3", "type": "arrow" }, { "id": "2ytpIbFV4FFvZCuZ9vq5v", "type": "arrow" }, { "id": "NAbvQ3uyiDDGNFnkkhY_W", "type": "arrow" }, { "id": "6JMqV2puJMwPKC7AqTrgn", "type": "arrow" }, { "id": "AaP19FDr7FhDj7XXwZ4Qy", "type": "arrow" }, { "id": "SrBArxlfE97WA0BrkmqEz", "type": "arrow" } ], "updated": 1794311380479, "created": 1794310204278, "link": null, "locked": false }, { "id": "zlYfXve9ohLAQ8wxFiw9d", "type": "rectangle", "x": -8720.678673777486, "y": -8824.22181149601, "width": 222.22216796875, "height": 663.3333892822266, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7E", "roundness": { "type": 3 }, "seed": 705173371, "version": 139, "versionNonce": 564063739, "isDeleted": false, "boundElements": [ { "id": "miOBTiXeIB2zMFpZQTLIk", "type": "arrow" }, { "id": "eo21HiWyZKO513s0EUTwI", "type": "arrow" }, { "id": "NAbvQ3uyiDDGNFnkkhY_W", "type": "arrow" }, { "id": "AaP19FDr7FhDj7XXwZ4Qy", "type": "arrow" } ], "updated": 1794311380480, "created": 1794310219205, "link": null, "locked": false }, { "id": "TYkM9NfwXVwNYLyuEhPw-", "type": "ellipse", "x": -8339.56758979311, "y": -8644.221796237221, "width": 258.8887939453125, "height": 242.2222442626953, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7G", "roundness": { "type": 2 }, "seed": 1754682267, "version": 107, "versionNonce": 181129563, "isDeleted": false, "boundElements": [ { "id": "2ivtZ93NGBDK050ZF6QI3", "type": "arrow" }, { "id": "2ytpIbFV4FFvZCuZ9vq5v", "type": "arrow" }, { "id": "6JMqV2puJMwPKC7AqTrgn", "type": "arrow" } ], "updated": 1794311380480, "created": 1794310226997, "link": null, "locked": false }, { "id": "fUiHDk6d0Nd9T1yrhKpWI", "type": "text", "x": -8680.678795847798, "y": -8806.7218201144, "width": 139.35635375976562, "height": 114.44445800781247, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7H", "roundness": null, "seed": 349266331, "version": 83, "versionNonce": 591862235, "isDeleted": false, "boundElements": [], "updated": 1794310746732, "created": 1794310268206, "link": null, "locked": false, "text": "Lease\nServer", "fontSize": 45.77778320312499, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Lease\nServer", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "ZJ45qoUAwqONc4sYE82LC", "type": "text", "x": -8697.945626003406, "y": -8688.885280424995, "width": 166.03247722229932, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7I", "roundness": null, "seed": 1926351419, "version": 284, "versionNonce": 1128941909, "isDeleted": false, "boundElements": [], "updated": 1794340554148, "created": 1794310286393, "link": null, "locked": false, "text": "Zookeeper, etcd", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Zookeeper, etcd", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "YnFi8XZajNwoHo4Jw2der", "type": "text", "x": -8694.907212749476, "y": -8653.636195911811, "width": 165.69984436035156, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7J", "roundness": null, "seed": 1791722165, "version": 206, "versionNonce": 1293818453, "isDeleted": false, "boundElements": [], "updated": 1794340556765, "created": 1794310301745, "link": null, "locked": false, "text": "Hands out leases\nand tokens", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Hands out leases\nand tokens", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "lH2ebmfjV6tpaYbk1HCeY", "type": "text", "x": -9080.234337839986, "y": -8558.944049118307, "width": 47.479949951171875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7K", "roundness": null, "seed": 134335099, "version": 6, "versionNonce": 671931803, "isDeleted": false, "boundElements": [], "updated": 1794310316076, "created": 1794310313848, "link": null, "locked": false, "text": "Node", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Node", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "0Xcv5IwhawyMyWVbEbvnp", "type": "text", "x": -8300.234337839986, "y": -8561.944049118307, "width": 76.29991149902344, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7L", "roundness": null, "seed": 1996532149, "version": 17, "versionNonce": 518353915, "isDeleted": false, "boundElements": [], "updated": 1794310336079, "created": 1794310323839, "link": null, "locked": false, "text": "Storage", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Storage", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "OpbUp4y3qsQuaOP0cOMrQ", "type": "text", "x": -9103.12325385561, "y": -8524.499530075338, "width": 137.9198760986328, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7M", "roundness": null, "seed": 1930629595, "version": 36, "versionNonce": 2031218491, "isDeleted": false, "boundElements": [], "updated": 1794310807720, "created": 1794310339319, "link": null, "locked": false, "text": "( Dispatcher )", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "( Dispatcher )", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "GoqezAVK3eRXnn04fBpUG", "type": "text", "x": -8312.234337839986, "y": -8527.944049118307, "width": 220.8397979736328, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7N", "roundness": null, "seed": 1697539221, "version": 25, "versionNonce": 235173269, "isDeleted": false, "boundElements": [], "updated": 1794310353826, "created": 1794310347879, "link": null, "locked": false, "text": "Database,file,job table", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Database,file,job table", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "miOBTiXeIB2zMFpZQTLIk", "type": "arrow", "x": -9008.198291643876, "y": -8615.277384005776, "width": 282.01961786639004, "height": 170.00042788217434, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7Q", "roundness": { "type": 2 }, "seed": 839914267, "version": 74, "versionNonce": 1701258965, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310401296, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 282.01961786639004, -170.00042788217434 ] ], "startBinding": { "elementId": "_AkzDK2j-D8XTAf663jTp", "mode": "orbit", "fixedPoint": [ 0.5001, 0 ] }, "endBinding": { "elementId": "zlYfXve9ohLAQ8wxFiw9d", "mode": "orbit", "fixedPoint": [ 0.044687256599173546, 0.044687256599169195 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "eo21HiWyZKO513s0EUTwI", "type": "arrow", "x": -8726.178673777484, "y": -8728.6727326206, "width": 198.65268619284143, "height": 114.05274166728668, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7R", "roundness": { "type": 2 }, "seed": 236047323, "version": 101, "versionNonce": 1696799995, "isDeleted": false, "boundElements": [ { "type": "text", "id": "lFN5fkeSz6f9zxbmkzJz2" } ], "updated": 1794334125384, "created": 1794310406225, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -82.27783203125182, 63.33982852182453 ], [ -198.65268619284143, 114.05274166728668 ] ], "startBinding": { "elementId": "zlYfXve9ohLAQ8wxFiw9d", "mode": "orbit", "fixedPoint": [ 0.10943710976607823, 0.10943710976607858 ] }, "endBinding": { "elementId": "_AkzDK2j-D8XTAf663jTp", "mode": "orbit", "fixedPoint": [ 0.27821475000262263, 0.27821475000262536 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "lFN5fkeSz6f9zxbmkzJz2", "type": "text", "x": -8867.254257531056, "y": -8684.845880698085, "width": 97.29991149902344, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7RV", "roundness": null, "seed": 328023157, "version": 22, "versionNonce": 436576923, "isDeleted": false, "boundElements": [], "updated": 1794311373490, "created": 1794310500533, "link": null, "locked": false, "text": "ok 10 sec,\ntoken 33", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "eo21HiWyZKO513s0EUTwI", "originalText": "ok 10 sec,\ntoken 33", "autoResize": true, "lineHeight": 1.25, "labelPosition": 0.5 }, { "id": "2ivtZ93NGBDK050ZF6QI3", "type": "arrow", "x": -8896.290001902484, "y": -8577.683836771193, "width": 563.3602405766142, "height": 2.093909109620654, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7S", "roundness": { "type": 2 }, "seed": 2119436763, "version": 199, "versionNonce": 1206223925, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310413737, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 563.3602405766142, 2.093909109620654 ] ], "startBinding": { "elementId": "_AkzDK2j-D8XTAf663jTp", "mode": "orbit", "fixedPoint": [ 0.8462730222236816, 0.15372697777631772 ] }, "endBinding": { "elementId": "TYkM9NfwXVwNYLyuEhPw-", "mode": "orbit", "fixedPoint": [ 0.5001, 0.28522737467516923 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "2ytpIbFV4FFvZCuZ9vq5v", "type": "arrow", "x": -8896.290001902486, "y": -8546.041128937273, "width": 554.354272903447, "height": 4.188695835578983, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7V", "roundness": { "type": 2 }, "seed": 32153819, "version": 90, "versionNonce": 910935451, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310428257, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 554.354272903447, -4.188695835578983 ] ], "startBinding": { "elementId": "_AkzDK2j-D8XTAf663jTp", "mode": "orbit", "fixedPoint": [ 0.6904463430701888, 0.3095536569298083 ] }, "endBinding": { "elementId": "TYkM9NfwXVwNYLyuEhPw-", "mode": "orbit", "fixedPoint": [ 0.5001, 0.38392758604323957 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "SrBArxlfE97WA0BrkmqEz", "type": "arrow", "x": -8726.178673777486, "y": -8437.237214797657, "width": 170.111328125, "height": 8.037116379749932, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7X", "roundness": { "type": 2 }, "seed": 1677577141, "version": 246, "versionNonce": 2037462421, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310440544, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -33.3890380859375, -3.6513533636170905 ], [ -170.111328125, -8.037116379749932 ] ], "startBinding": null, "endBinding": { "elementId": "_AkzDK2j-D8XTAf663jTp", "mode": "orbit", "fixedPoint": [ 0.7831390668981788, 0.7831390668981888 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "6JMqV2puJMwPKC7AqTrgn", "type": "arrow", "x": -8905.603565684116, "y": -8406.412360669154, "width": 631.3056625299087, "height": 5.320905727478021, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7Y", "roundness": { "type": 2 }, "seed": 1022103835, "version": 118, "versionNonce": 2039896635, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310462776, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 631.3056625299087, -5.320905727478021 ] ], "startBinding": { "elementId": "_AkzDK2j-D8XTAf663jTp", "mode": "orbit", "fixedPoint": [ 0.9691234538616553, 0.9788979728960517 ] }, "endBinding": { "elementId": "TYkM9NfwXVwNYLyuEhPw-", "mode": "orbit", "fixedPoint": [ 0.5001, 0.9575809054403022 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "AaP19FDr7FhDj7XXwZ4Qy", "type": "arrow", "x": -8956.708398437733, "y": -8396.499582492104, "width": 230.52972466024767, "height": 99.93857504990228, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7a", "roundness": { "type": 2 }, "seed": 1464036981, "version": 97, "versionNonce": 1430239989, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310477533, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 230.52972466024767, 99.93857504990228 ] ], "startBinding": { "elementId": "_AkzDK2j-D8XTAf663jTp", "mode": "orbit", "fixedPoint": [ 0.2301617294271841, 0.7698382705728289 ] }, "endBinding": { "elementId": "zlYfXve9ohLAQ8wxFiw9d", "mode": "orbit", "fixedPoint": [ 0.17545533471186334, 0.8245446652881384 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "pqhCpNo2xPG8CyOWJOKe9", "type": "text", "x": -8916.09537649987, "y": -8728.342055675192, "width": 89.25991821289062, "height": 25, "angle": 5.778011433018238, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7b", "roundness": null, "seed": 1929650491, "version": 95, "versionNonce": 28568091, "isDeleted": false, "boundElements": [], "updated": 1794340528007, "created": 1794310496065, "link": null, "locked": false, "text": "get lease", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "get lease", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "OaAQe1lsKc4gMjA1W8SI2", "type": "text", "x": -8876.863876661328, "y": -8374.726637733434, "width": 66.93994140624999, "height": 25, "angle": 0.3553334135180144, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7d", "roundness": null, "seed": 1168975893, "version": 45, "versionNonce": 224780085, "isDeleted": false, "boundElements": [], "updated": 1794340521552, "created": 1794310583979, "link": null, "locked": false, "text": "release", "fontSize": 19.999999999999996, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "release", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "yHTFuBU_zEJ2QmtXlvTjt", "type": "text", "x": -8801.345299754048, "y": -8608.166247604635, "width": 49.179931640625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7e", "roundness": null, "seed": 985645141, "version": 17, "versionNonce": 164317051, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310600043, "link": null, "locked": false, "text": "write", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "write", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "4ftXFejYmKbc6iI_qTaIU", "type": "text", "x": -8794.78975776186, "y": -8572.610736130026, "width": 49.179931640625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7f", "roundness": null, "seed": 1133390005, "version": 19, "versionNonce": 1091256757, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310604331, "link": null, "locked": false, "text": "write", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "write", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "YE2AnqeBp0fy-f92SitZl", "type": "text", "x": -8831.012047800923, "y": -8507.944049118307, "width": 53.699951171875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7g", "roundness": null, "seed": 1256858299, "version": 8, "versionNonce": 428515355, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310612003, "link": null, "locked": false, "text": "renew", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "renew", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "M-EesMNh82mLnoO3Vajfx", "type": "text", "x": -8851.567467722798, "y": -8467.499591110494, "width": 115.09988403320312, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7h", "roundness": null, "seed": 457490133, "version": 43, "versionNonce": 302203669, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310619563, "link": null, "locked": false, "text": "ok,token 33", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "ok,token 33", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "NAbvQ3uyiDDGNFnkkhY_W", "type": "arrow", "x": -8896.290001902484, "y": -8483.134700848255, "width": 170.11132812499818, "height": 2.819939113867804, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7i", "roundness": { "type": 2 }, "seed": 1559376795, "version": 123, "versionNonce": 1964369083, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310437208, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 170.11132812499818, -2.819939113867804 ] ], "startBinding": { "elementId": "_AkzDK2j-D8XTAf663jTp", "mode": "orbit", "fixedPoint": [ 0.6170107619304213, 0.6170107619304354 ] }, "endBinding": { "elementId": "zlYfXve9ohLAQ8wxFiw9d", "mode": "orbit", "fixedPoint": [ 0.4929244065142119, 0.5070755934857798 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "_WN48u0qjNmOyNij6631k", "type": "text", "x": -8816.900963816548, "y": -8433.055133102682, "width": 49.179931640625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7k", "roundness": null, "seed": 1548632699, "version": 23, "versionNonce": 2067011701, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310638874, "link": null, "locked": false, "text": "write", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "write", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "P00LG8gPQ1acK5ioZIMbo", "type": "text", "x": -9132.345299754048, "y": -8892.721806833602, "width": 336.439697265625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7l", "roundness": null, "seed": 716048987, "version": 67, "versionNonce": 336565595, "isDeleted": false, "boundElements": [], "updated": 1794334125384, "created": 1794310947166, "link": null, "locked": false, "text": "here, token means `fencing token`", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "here, token means `fencing token`", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "HMDzqLbUU57MkK-DyqnfF", "type": "text", "x": -7444.156672970284, "y": -8935.718341847467, "width": 588.5594482421875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7q", "roundness": null, "seed": 1472922395, "version": 154, "versionNonce": 1317128917, "isDeleted": false, "boundElements": [], "updated": 1794336462698, "created": 1794333686405, "link": null, "locked": false, "text": "The silence from the workers can have any of these causes:", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "The silence from the workers can have any of these causes:", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "VjJcrTJdx-tsl61eqo00c", "type": "text", "x": -7431.2382502152495, "y": -8898.120518147325, "width": 587.9909304890951, "height": 21.472542258346948, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7r", "roundness": null, "seed": 548311061, "version": 368, "versionNonce": 1154709333, "isDeleted": false, "boundElements": [], "updated": 1794333794810, "created": 1794333686405, "link": null, "locked": false, "text": "Cause Worker state Job state", "fontSize": 17.178033806677558, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Cause Worker state Job state", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "U2dnAHWKXdfY_CcII7Nu0", "type": "text", "x": -7487.941498463707, "y": -8868.411720907176, "width": 762.6195678710938, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7s", "roundness": null, "seed": 1934747579, "version": 36, "versionNonce": 1676117813, "isDeleted": false, "boundElements": [], "updated": 1794333791063, "created": 1794333686405, "link": null, "locked": false, "text": "Worker crashed Dead Container dead too", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Worker crashed Dead Container dead too", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "bQDCUsb0Au7U3XFDUyTyP", "type": "text", "x": -7640.098212385017, "y": -8842.076338178133, "width": 942.3936614294371, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7t", "roundness": null, "seed": 284018037, "version": 185, "versionNonce": 396037339, "isDeleted": false, "boundElements": [ { "id": "0KRg4zmyv6pKaQwH_JPl6", "type": "arrow" } ], "updated": 1794334173892, "created": 1794333686407, "link": null, "locked": false, "text": "Worker paused (GC, VM stop, swap) Alive, will wake up Container may still run", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Worker paused (GC, VM stop, swap) Alive, will wake up Container may still run", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "sKCGkFsvmamn4v-VG8Hvq", "type": "text", "x": -7488.495179667197, "y": -8814.565265380836, "width": 756.0994873046875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7u", "roundness": null, "seed": 299055195, "version": 96, "versionNonce": 1317257531, "isDeleted": false, "boundElements": [], "updated": 1794333772455, "created": 1794333686407, "link": null, "locked": false, "text": "Network partition Alive, running normally Container still runs", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Network partition Alive, running normally Container still runs", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "SNFZ4AcWvVz5ZEW_3PKi3", "type": "text", "x": -7576.330326232997, "y": -8776.625765704905, "width": 878.1917700241697, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7v", "roundness": null, "seed": 308235989, "version": 135, "versionNonce": 343052085, "isDeleted": false, "boundElements": [], "updated": 1794333954336, "created": 1794333686407, "link": null, "locked": false, "text": "Worker is slow, heartbeat late Alive Container still runs", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Worker is slow, heartbeat late Alive Container still runs", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "hmqqZZubV0Od8g9TPFODq", "type": "line", "x": -7599.786654132674, "y": -8869.989697331815, "width": 871.8674043801166, "height": 13.521872900893868, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7x", "roundness": { "type": 2 }, "seed": 1733368981, "version": 108, "versionNonce": 1669962325, "isDeleted": false, "boundElements": [], "updated": 1794333808773, "created": 1794333806692, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 871.8674043801166, -13.521872900893868 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "bNHZpNWOCbuE9IxmJKVcK", "type": "line", "x": -7252.585766938194, "y": -8906.793426267272, "width": 8.48338550983317, "height": 207.37496689880572, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7y", "roundness": { "type": 2 }, "seed": 1135591099, "version": 84, "versionNonce": 1757793179, "isDeleted": false, "boundElements": [], "updated": 1794336465485, "created": 1794333810948, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 8.48338550983317, 207.37496689880572 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "6pFteyWKzIwI2aG_oWj2Q", "type": "line", "x": -6962.494559825067, "y": -8905.768891447171, "width": 0.9232608085176253, "height": 204.98321892979584, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b7z", "roundness": { "type": 2 }, "seed": 1341577045, "version": 70, "versionNonce": 1834443131, "isDeleted": false, "boundElements": [], "updated": 1794336471654, "created": 1794333813108, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0.9232608085176253, 204.98321892979584 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "64V-mspySGye-rPsQnjHE", "type": "line", "x": -7643.8797467243785, "y": -8848.825015471377, "width": 947.7075197624799, "height": 2.3516384944359743, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b86", "roundness": { "type": 2 }, "seed": 1309519803, "version": 138, "versionNonce": 683889851, "isDeleted": false, "boundElements": [], "updated": 1794333832666, "created": 1794333828083, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 947.7075197624799, 2.3516384944359743 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "GtHBWzTL3JfStlXn7aFbs", "type": "line", "x": -7663.280715861361, "y": -8815.314279957263, "width": 987.0972546284593, "height": 3.5274254469113657, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b87", "roundness": { "type": 2 }, "seed": 972754677, "version": 99, "versionNonce": 1750117461, "isDeleted": false, "boundElements": [], "updated": 1794333854326, "created": 1794333852531, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 987.0972546284593, -3.5274254469113657 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "ESQ0YkWzbJ89OG8sfiNDK", "type": "line", "x": -7650.934646060316, "y": -8784.743076413823, "width": 960.6414830397653, "height": 3.5274254469113657, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b88", "roundness": { "type": 2 }, "seed": 1328499899, "version": 95, "versionNonce": 1392656091, "isDeleted": false, "boundElements": [], "updated": 1794333857612, "created": 1794333855523, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 960.6414830397653, -3.5274254469113657 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "YAGiyvE_mYCXLXmX-araR", "type": "line", "x": -7638.00066663566, "y": -8734.770903733399, "width": 952.410780603981, "height": 10.582340930219289, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b89", "roundness": { "type": 2 }, "seed": 1651482389, "version": 93, "versionNonce": 790504501, "isDeleted": false, "boundElements": [], "updated": 1794333860895, "created": 1794333858610, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 952.410780603981, -10.582340930219289 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "fnnQ6FfjQQ_kyV4XReAea", "type": "arrow", "x": -6621.985514161803, "y": -8755.616801603961, "width": 68.50124494454394, "height": 46.26838068966208, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8E", "roundness": { "type": 2 }, "seed": 1461872565, "version": 120, "versionNonce": 94487861, "isDeleted": false, "boundElements": [], "updated": 1794334182859, "created": 1794334132008, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0.6008730573821595, 46.26838068966208 ], [ -67.90037188716178, 31.24619116986105 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "pE9lKTRF-VzWNNVeFH7s_", "type": "text", "x": -6660.141943538056, "y": -8801.526155585047, "width": 183.13983154296875, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8H", "roundness": null, "seed": 1302777659, "version": 45, "versionNonce": 1137005531, "isDeleted": false, "boundElements": [ { "id": "0KRg4zmyv6pKaQwH_JPl6", "type": "arrow" } ], "updated": 1794334186244, "created": 1794334152310, "link": null, "locked": false, "text": "Fencing is required\n for these 3 cases", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Fencing is required\n for these 3 cases", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "0KRg4zmyv6pKaQwH_JPl6", "type": "arrow", "x": -6627.39350370924, "y": -8798.880684975711, "width": 63.09325539710608, "height": 44.46569550201639, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8I", "roundness": { "type": 2 }, "seed": 1035534907, "version": 121, "versionNonce": 940356533, "isDeleted": false, "boundElements": [], "updated": 1794334187571, "created": 1794334129193, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -1.8026191721482974, -42.663010314372514 ], [ -63.09325539710608, -44.46569550201639 ] ], "startBinding": { "elementId": "pE9lKTRF-VzWNNVeFH7s_", "mode": "inside", "fixedPoint": [ 0.17881658813873372, 0.052909412186709234 ] }, "endBinding": { "elementId": "bQDCUsb0Au7U3XFDUyTyP", "mode": "orbit", "fixedPoint": [ 1.0076589987227698, -0.05080169198379735 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "ZoKDswAURN1NiZpnsziuf", "type": "text", "x": -7607.568222428393, "y": -8614.705946251503, "width": 1012.8206118586176, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8K", "roundness": null, "seed": 1661333173, "version": 350, "versionNonce": 1448284411, "isDeleted": false, "boundElements": [], "updated": 1794335718768, "created": 1794334297771, "link": null, "locked": false, "text": "Setting What it measures Renewed by Event", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Setting What it measures Renewed by Event", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "0mW6oDZMlparsz329PgjS", "type": "text", "x": -7629.648551077602, "y": -8584.160714255944, "width": 1043.2564508962498, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8L", "roundness": null, "seed": 731726619, "version": 262, "versionNonce": 1072485877, "isDeleted": false, "boundElements": [ { "id": "YFVCQldyRojPWPuX2MeVJ", "type": "arrow" } ], "updated": 1794335752757, "created": 1794334297771, "link": null, "locked": false, "text": "lease_timeout Time since the last heartbeat Every heartbeat(job_id) LeaseExpired", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "lease_timeout Time since the last heartbeat Every heartbeat(job_id) LeaseExpired", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "Mqf-uWsRG5fNs8VHTsVX1", "type": "text", "x": -7623.638340637867, "y": -8552.082780514711, "width": 1012.0398138059292, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8M", "roundness": null, "seed": 1512067093, "version": 226, "versionNonce": 1276581493, "isDeleted": false, "boundElements": [], "updated": 1794334392628, "created": 1794334297771, "link": null, "locked": false, "text": "job_timeout Time since the job started Never, it is a hard limit TimedOut", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "job_timeout Time since the job started Never, it is a hard limit TimedOut", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "pb1PsVgY2GeUdeiYTVJVH", "type": "line", "x": -7653.050857160136, "y": -8591.64639297705, "width": 1018.6883019232009, "height": 3.105783076825901, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8W", "roundness": { "type": 2 }, "seed": 25672155, "version": 221, "versionNonce": 492069979, "isDeleted": false, "boundElements": [], "updated": 1794334505164, "created": 1794334448120, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1018.6883019232009, 3.105783076825901 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "QOTxFxryAjeu_jl6cllaO", "type": "line", "x": -6958.603674327083, "y": -8593.509835526458, "width": 8.074974582196774, "height": 1.2423064065569633, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8X", "roundness": { "type": 2 }, "seed": 1996077365, "version": 9, "versionNonce": 892671701, "isDeleted": false, "boundElements": [], "updated": 1794334450138, "created": 1794334450056, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 8.074974582196774, -1.2423064065569633 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "DGXbZzLqb-64U2Vsm3EC4", "type": "line", "x": -7658.020125437445, "y": -8557.172501087858, "width": 1018.6883019232009, "height": 3.105783076825901, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8b", "roundness": { "type": 2 }, "seed": 1461567131, "version": 228, "versionNonce": 802239387, "isDeleted": false, "boundElements": [], "updated": 1794334540093, "created": 1794334533537, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1018.6883019232009, 3.105783076825901 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "xXUoA6sDX02bk8SDfx-sH", "type": "line", "x": -7648.081742426701, "y": -8523.009262572245, "width": 1018.6883019232009, "height": 3.105783076825901, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8c", "roundness": { "type": 2 }, "seed": 426199259, "version": 330, "versionNonce": 1648981019, "isDeleted": false, "boundElements": [], "updated": 1794334551171, "created": 1794334542315, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1018.6883019232009, 3.105783076825901 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "cV-P2h20Fw_HDh64pTIdj", "type": "line", "x": -7471.674650675993, "y": -8621.461644371862, "width": 1.2423064065578728, "height": 127.33601428230759, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8d", "roundness": { "type": 2 }, "seed": 1915316571, "version": 56, "versionNonce": 1250567963, "isDeleted": false, "boundElements": [], "updated": 1794334559980, "created": 1794334558246, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -1.2423064065578728, 127.33601428230759 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "qMCkhTSIBLHlqVwYu3z4j", "type": "line", "x": -7107.679965422008, "y": -8615.871282602782, "width": 6.211497911930564, "height": 126.09374199661033, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8e", "roundness": { "type": 2 }, "seed": 469592789, "version": 35, "versionNonce": 973354037, "isDeleted": false, "boundElements": [], "updated": 1794334561282, "created": 1794334560718, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -6.211497911930564, 126.09374199661033 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "MKiluRdJhNEheb3ZrUPa-", "type": "line", "x": -6802.073544792811, "y": -8622.703950778421, "width": 5.590361769081028, "height": 136.03215912821543, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8g", "roundness": { "type": 2 }, "seed": 735773787, "version": 57, "versionNonce": 1034454715, "isDeleted": false, "boundElements": [], "updated": 1794334566468, "created": 1794334564478, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 5.590361769081028, 136.03215912821543 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "6HVt_mvE8WsQvGLj9NuGm", "type": "text", "x": -7609.961433331581, "y": -8418.919839335726, "width": 920.1027678964898, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8h", "roundness": null, "seed": 2025189851, "version": 191, "versionNonce": 856643195, "isDeleted": false, "boundElements": [], "updated": 1794336374959, "created": 1794334638632, "link": null, "locked": false, "text": "lease_expires_at = last_heartbeat + lease_timeout -- moves forward on each heartbeat", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "lease_expires_at = last_heartbeat + lease_timeout -- moves forward on each heartbeat", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "aoBliA_JG_sfcCIZABLvA", "type": "text", "x": -7616.271056726335, "y": -8371.91095121671, "width": 897.8314566394357, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8i", "roundness": null, "seed": 188673365, "version": 191, "versionNonce": 1560826491, "isDeleted": false, "boundElements": [], "updated": 1794336398046, "created": 1794334638632, "link": null, "locked": false, "text": "timeout_at = started_at + job_timeout -- fixed when the job starts", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "timeout_at = started_at + job_timeout -- fixed when the job starts", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "ivzQ4wu7yIuLhZZ08-GQE", "type": "line", "x": -7637.35485833894, "y": -8381.259891334259, "width": 952.4504562519624, "height": 0.19212273687480774, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8j", "roundness": { "type": 2 }, "seed": 1939239739, "version": 194, "versionNonce": 1982948571, "isDeleted": false, "boundElements": [], "updated": 1794336402941, "created": 1794334649509, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 952.4504562519624, 0.19212273687480774 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "kc0d2hm0jXxEdpSFWWG57", "type": "line", "x": -7633.251839842429, "y": -8428.209720564093, "width": 944.1831609409401, "height": 6.894056243871091, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8m", "roundness": { "type": 2 }, "seed": 805959323, "version": 239, "versionNonce": 1969187797, "isDeleted": false, "boundElements": [], "updated": 1794336394140, "created": 1794334662739, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 944.1831609409401, -6.894056243871091 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "6SB8qR1YOjtorXXd5eAPS", "type": "line", "x": -7642.140175666931, "y": -8346.4961739924, "width": 961.8983488923459, "height": 1.372979566693175, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8n", "roundness": { "type": 2 }, "seed": 378342229, "version": 252, "versionNonce": 121107317, "isDeleted": false, "boundElements": [], "updated": 1794336401077, "created": 1794334667315, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 961.8983488923459, 1.372979566693175 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "CJt1x9Zp-kUP-_xYpTbcc", "type": "rectangle", "x": -7023.823108741706, "y": -9270.375785904054, "width": 119.10719646910002, "height": 147.7449488715829, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8q", "roundness": { "type": 3 }, "seed": 469141685, "version": 124, "versionNonce": 78000379, "isDeleted": false, "boundElements": [ { "id": "ez1KHpXfYNhSgszrXZ07L", "type": "arrow" } ], "updated": 1794335106751, "created": 1794334917748, "link": null, "locked": false }, { "id": "gBfJXZkqHPHRbIlJ6vGOE", "type": "rectangle", "x": -7376.588654300822, "y": -9263.867213125643, "width": 143.18895865251852, "height": 150.34842803679203, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8r", "roundness": { "type": 3 }, "seed": 535306651, "version": 106, "versionNonce": 1141490107, "isDeleted": false, "boundElements": [ { "id": "ez1KHpXfYNhSgszrXZ07L", "type": "arrow" }, { "id": "Em6ISeTE5czmRGLGX6vqI", "type": "arrow" } ], "updated": 1794335397515, "created": 1794334920989, "link": null, "locked": false }, { "id": "8JTlq6cKtQ6V-NmP8s6jK", "type": "text", "x": -7369.168887053871, "y": -9212.452870414501, "width": 105.05989074707031, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8s", "roundness": null, "seed": 1952434587, "version": 12, "versionNonce": 1927176315, "isDeleted": false, "boundElements": [], "updated": 1794334929790, "created": 1794334926677, "link": null, "locked": false, "text": "Dispatcher", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Dispatcher", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Qpmgs8Xr2cxewQ_j7Lm2v", "type": "text", "x": -6994.729741228581, "y": -9240.374709843667, "width": 64.99992370605469, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8t", "roundness": null, "seed": 1148014171, "version": 64, "versionNonce": 1951183067, "isDeleted": false, "boundElements": [], "updated": 1794335098294, "created": 1794334931459, "link": null, "locked": false, "text": " Pull\nbased\nWorker\n (sm)", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": " Pull\nbased\nWorker\n (sm)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "ez1KHpXfYNhSgszrXZ07L", "type": "arrow", "x": -7029.323108741707, "y": -9191.47074672016, "width": 198.57658690659719, "height": 2.717515107613508, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8u", "roundness": { "type": 2 }, "seed": 1417233237, "version": 113, "versionNonce": 474765723, "isDeleted": false, "boundElements": [], "updated": 1794335106752, "created": 1794334966831, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -198.57658690659719, 2.717515107613508 ] ], "startBinding": { "elementId": "CJt1x9Zp-kUP-_xYpTbcc", "mode": "orbit", "fixedPoint": [ 0.4716503514468348, 0.528349648553165 ] }, "endBinding": { "elementId": "gBfJXZkqHPHRbIlJ6vGOE", "mode": "orbit", "fixedPoint": [ 1, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "GmriA_midJ-mrXg1WVRyn", "type": "text", "x": -7199.0344428191665, "y": -9276.106861633847, "width": 111.139892578125, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8v", "roundness": null, "seed": 41793499, "version": 27, "versionNonce": 490253819, "isDeleted": false, "boundElements": [], "updated": 1794334988883, "created": 1794334973708, "link": null, "locked": false, "text": " API call\nlease (size)", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": " API call\nlease (size)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "C3Uu5kyz1BgqqE-cUsy1v", "type": "text", "x": -7190.833621096832, "y": -9178.738353303872, "width": 154.95138198065703, "height": 175, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8w", "roundness": null, "seed": 1187705685, "version": 364, "versionNonce": 341094811, "isDeleted": false, "boundElements": [], "updated": 1794335073871, "created": 1794335007567, "link": null, "locked": false, "text": "Pull based\nworker, once\nfree sends the\nAPI call\nrequesting the\ndispatcher for\na job", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Pull based worker, once\nfree sends the API call requesting the dispatcher for a job", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "Em6ISeTE5czmRGLGX6vqI", "type": "arrow", "x": -7382.088654300822, "y": -9188.825784593062, "width": 128.25149304143997, "height": 3.4469414267896354, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8x", "roundness": { "type": 2 }, "seed": 1838229851, "version": 233, "versionNonce": 756526197, "isDeleted": false, "boundElements": [], "updated": 1794335423841, "created": 1794335328105, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -128.25149304143997, -3.4469414267896354 ] ], "startBinding": { "elementId": "gBfJXZkqHPHRbIlJ6vGOE", "mode": "orbit", "fixedPoint": [ 0, 0.5001 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "xuBaO0jN2pyWSjHVabiEE", "type": "text", "x": -7394.300564483697, "y": -9511.978136995474, "width": 39.979848572106775, "height": 14.591198611368709, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8y", "roundness": null, "seed": 255849915, "version": 218, "versionNonce": 935320091, "isDeleted": false, "boundElements": [], "updated": 1794335504584, "created": 1794335334087, "link": null, "locked": false, "text": "BEGIN", "fontSize": 11.672958889094968, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "BEGIN", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "DxGdb4DobVqk-sCR0GSMl", "type": "text", "x": -7585.987894302509, "y": -9497.920175021682, "width": 423.35450820973097, "height": 29.18239722273742, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b8z", "roundness": null, "seed": 2100270965, "version": 230, "versionNonce": 1678501083, "isDeleted": false, "boundElements": [], "updated": 1794337808287, "created": 1794335334088, "link": null, "locked": false, "text": "pick the lowest-tag Queued job with size <= sm and limit 1 -- FOR\nUPDATE SKIP LOCKED", "fontSize": 11.672958889094968, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "pick the lowest-tag Queued job with size <= sm and limit 1 -- FOR UPDATE SKIP LOCKED", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "LTDksppg2GeXD5UjhFXmc", "type": "text", "x": -7480.569494482544, "y": -9455.605699048712, "width": 212.51770856980207, "height": 14.591198611368709, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b90", "roundness": null, "seed": 1624213083, "version": 217, "versionNonce": 2109624155, "isDeleted": false, "boundElements": [], "updated": 1794335504584, "created": 1794335334088, "link": null, "locked": false, "text": "UPDATE jobs SET state = 'Running',", "fontSize": 11.672958889094968, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "UPDATE jobs SET state = 'Running',", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "QF8swpGwmOo4_LiXPZ2KC", "type": "text", "x": -7407.613563766026, "y": -9435.178020992797, "width": 66.60584713676717, "height": 14.591198611368709, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b91", "roundness": null, "seed": 2056708309, "version": 217, "versionNonce": 1301392379, "isDeleted": false, "boundElements": [], "updated": 1794335504584, "created": 1794335334088, "link": null, "locked": false, "text": "worker = 'A',", "fontSize": 11.672958889094968, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "worker = 'A',", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "R7STjSiKhe7sVPwsC7nBT", "type": "text", "x": -7430.054798731876, "y": -9414.75034293688, "width": 111.32919311523438, "height": 14.59119861136871, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b92", "roundness": null, "seed": 1197736699, "version": 218, "versionNonce": 679306933, "isDeleted": false, "boundElements": [], "updated": 1794335518469, "created": 1794335334088, "link": null, "locked": false, "text": "attempt = attempt,", "fontSize": 11.672958889094968, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "attempt = attempt,", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "QDt_DAGErXQIR3wn8lEu3", "type": "text", "x": -7493.468099805776, "y": -9394.322664880965, "width": 238.31491921626713, "height": 14.591198611368709, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b93", "roundness": null, "seed": 1393425973, "version": 217, "versionNonce": 1961699643, "isDeleted": false, "boundElements": [], "updated": 1794335504584, "created": 1794335334088, "link": null, "locked": false, "text": "lease_expires_at = now() + lease_timeout", "fontSize": 11.672958889094964, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "lease_expires_at = now() + lease_timeout", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "fJz-DdrKxZAyyyltEfn6c", "type": "text", "x": -7511.281028658388, "y": -9373.89498682505, "width": 273.9407769214893, "height": 14.591198611368709, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b94", "roundness": null, "seed": 729803675, "version": 217, "versionNonce": 294275547, "isDeleted": false, "boundElements": [], "updated": 1794335504584, "created": 1794335334088, "link": null, "locked": false, "text": "UPDATE scheduler_state SET virtual_now = ...", "fontSize": 11.672958889094968, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "UPDATE scheduler_state SET virtual_now = ...", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "G1m8eUrJzw5sHgd3gvOYu", "type": "text", "x": -7399.582575709283, "y": -9353.467308769134, "width": 50.543871023281206, "height": 14.591198611368709, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b95", "roundness": null, "seed": 1031434133, "version": 217, "versionNonce": 721049211, "isDeleted": false, "boundElements": [], "updated": 1794335504584, "created": 1794335334088, "link": null, "locked": false, "text": "COMMIT", "fontSize": 11.672958889094968, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "COMMIT", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "t-886SUvgFyulUD2zIxDz", "type": "text", "x": -7499.82954174092, "y": -9323.708696259706, "width": 219.28112403155023, "height": 16.765650869837174, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b96", "roundness": null, "seed": 1096138811, "version": 175, "versionNonce": 942183291, "isDeleted": false, "boundElements": [], "updated": 1794335512876, "created": 1794335334088, "link": null, "locked": false, "text": "reply to worker: job 17, attempt 1", "fontSize": 13.412520695869736, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "reply to worker: job 17, attempt 1", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "WDb7D9wNEu5xN1HQ1l3r-", "type": "rectangle", "x": -7678.261746907689, "y": -9256.056875108723, "width": 164.0164344467321, "height": 147.09408801846803, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b97", "roundness": { "type": 3 }, "seed": 452825659, "version": 79, "versionNonce": 207719195, "isDeleted": false, "boundElements": [], "updated": 1794335352575, "created": 1794335347832, "link": null, "locked": false }, { "id": "V-WKUYAJLxQ6EgRe-qxYG", "type": "text", "x": -7666.676498802993, "y": -9243.563918456766, "width": 140.099853515625, "height": 125, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b99", "roundness": null, "seed": 1104258459, "version": 68, "versionNonce": 863506549, "isDeleted": false, "boundElements": [], "updated": 1794335389808, "created": 1794335356236, "link": null, "locked": false, "text": "Persistent\nSingle Source\nof truth\n(Database,\nFiles, storage)", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Persistent\nSingle Source\nof truth\n(Database,\nFiles, storage)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Kz3RC8l9j7tVN5APzqPK1", "type": "text", "x": -7488.601512832256, "y": -9248.835875278268, "width": 81.2799072265625, "height": 125, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9B", "roundness": null, "seed": 943931675, "version": 36, "versionNonce": 88131323, "isDeleted": false, "boundElements": [], "updated": 1794335446895, "created": 1794335430207, "link": null, "locked": false, "text": "After \nreceiving\n\nthe API\ncall", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "After \nreceiving\n\nthe API\ncall", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "SnnbQ_a1n1j1-XsDMPtem", "type": "line", "x": -7451.112006348829, "y": -9249.548302330311, "width": 0, "height": 38.400665199218565, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9C", "roundness": { "type": 2 }, "seed": 2143046229, "version": 42, "versionNonce": 525422933, "isDeleted": false, "boundElements": [], "updated": 1794335458849, "created": 1794335456419, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, -38.400665199218565 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "vYT40Ykii54pVbn9fuN1g", "type": "rectangle", "x": -7591.0466786121415, "y": -9521.607312150452, "width": 438.02794191345146, "height": 227.80061481925424, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9E", "roundness": { "type": 3 }, "seed": 1883712507, "version": 170, "versionNonce": 1529344155, "isDeleted": false, "boundElements": [], "updated": 1794335517849, "created": 1794335477291, "link": null, "locked": false }, { "id": "izndS2ap93pGwFrg7Dz1B", "type": "arrow", "x": -6555.530153917472, "y": -8534.905218359798, "width": 70.94360059677092, "height": 26.685205595882508, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9J", "roundness": { "type": 2 }, "seed": 261529429, "version": 68, "versionNonce": 69434965, "isDeleted": false, "boundElements": [], "updated": 1794335744271, "created": 1794335736447, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -11.715459603337877, 25.383519642395186 ], [ -70.94360059677092, 26.685205595882508 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "YFVCQldyRojPWPuX2MeVJ", "type": "arrow", "x": -6561.713305207512, "y": -8572.837354167086, "width": 61.83151290040496, "height": 16.92231067551984, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9L", "roundness": { "type": 2 }, "seed": 162535765, "version": 210, "versionNonce": 1325754811, "isDeleted": false, "boundElements": [], "updated": 1794335762908, "created": 1794335750893, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -10.942915942673615, -9.111980438132377 ], [ -61.83151290040496, -16.92231067551984 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "Z6N0RANyDDB8Rot0MA9uf", "type": "text", "x": -6596.859755523017, "y": -8563.351300322947, "width": 261.69976806640625, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9M", "roundness": null, "seed": 1005951573, "version": 28, "versionNonce": 1193234293, "isDeleted": false, "boundElements": [], "updated": 1794335772677, "created": 1794335765811, "link": null, "locked": false, "text": "both case fencing needed !", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "both case fencing needed !", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "eYmFsB3tTyU0XuSgnOiJ0", "type": "text", "x": -7620.002108754914, "y": -10207.230836974897, "width": 356.0596923828125, "height": 125, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9N", "roundness": null, "seed": 999552379, "version": 356, "versionNonce": 370127125, "isDeleted": false, "boundElements": [], "updated": 1794337913885, "created": 1794335852676, "link": null, "locked": false, "text": "The problem we are fighting against:\n1. Worker crashed/dead\n2. Worker paused\n3. Network partition\n4. slow worker, heartbeat is late", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "The problem we are fighting against:\n1. Worker crashed/dead\n2. Worker paused\n3. Network partition\n4. slow worker, heartbeat is late", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "x6NI0h9TMX5D2QmTuMssp", "type": "text", "x": -7630.968008592535, "y": -10048.251290553875, "width": 1187.3729402722126, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9O", "roundness": null, "seed": 956065115, "version": 737, "versionNonce": 2031376885, "isDeleted": false, "boundElements": [], "updated": 1794337915285, "created": 1794335950404, "link": null, "locked": false, "text": "** Why fencing needed ?\n- because in the last 2,3,4 cases above, the dispatcher does not know the worker is alive, but due to lease_timeout\nanother worker takes place and 2 workers does the same job concurrently and can cause dual writes for the same job on\nthe database, so to prevent this we need the `fencing token`", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "** Why fencing needed ?\n- because in the last 2,3,4 cases above, the dispatcher does not know the worker is alive, but due to lease_timeout another worker takes place and 2 workers does the same job concurrently and can cause dual writes for the same job on the database, so to prevent this we need the `fencing token`", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "eafOyl-CYv4arUUWXGV25", "type": "text", "x": -7622.888420672398, "y": -9920.072426753315, "width": 800, "height": 225, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9P", "roundness": null, "seed": 1227025653, "version": 85, "versionNonce": 1574818581, "isDeleted": false, "boundElements": [], "updated": 1794337916777, "created": 1794336172486, "link": null, "locked": false, "text": "2 cases:\nSo the dispatcher sends the kill and then has to choose:\n\n1. Wait for confirmation. Safe, no fencing needed. But a network partitioned\nworker can hold the job in limbo. How long do you wait? You need a timeout on\nthe wait. That timeout is a lease timeout by another name, and you are back in\nthe lease case.\n2. Requeue after a bounded wait. Fast, but the old container may still be alive.\nNow fencing is needed.", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "2 cases:\nSo the dispatcher sends the kill and then has to choose:\n\n1. Wait for confirmation. Safe, no fencing needed. But a network partitioned worker can hold the job in limbo. How long do you wait? You need a timeout on the wait. That timeout is a lease timeout by another name, and you are back in the lease case.\n2. Requeue after a bounded wait. Fast, but the old container may still be alive. Now fencing is needed.", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "7Ir4QOPui4WKnJymS_Nlp", "type": "line", "x": -7157.477147128088, "y": -9406.58723073994, "width": 136.44340025316978, "height": 2.598999409468888, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9Q", "roundness": { "type": 2 }, "seed": 778121269, "version": 43, "versionNonce": 533824379, "isDeleted": false, "boundElements": [], "updated": 1794336696708, "created": 1794336690072, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 136.44340025316978, 2.598999409468888 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "VnJLzglJJvQoUsDDtuZi3", "type": "rectangle", "x": -7014.536462495664, "y": -9674.043279921369, "width": 642.7070130187786, "height": 370.47808000881906, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9T", "roundness": { "type": 3 }, "seed": 1252957301, "version": 312, "versionNonce": 621925979, "isDeleted": false, "boundElements": [], "updated": 1794338021431, "created": 1794336702998, "link": null, "locked": false }, { "id": "mpUKKhTqbbK6IFOa8DWxK", "type": "text", "x": -7006.043344381342, "y": -9626.530805228509, "width": 623.1083058410336, "height": 249.25799806875358, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9V", "roundness": null, "seed": 1386101819, "version": 528, "versionNonce": 1344340571, "isDeleted": false, "boundElements": [], "updated": 1794337977606, "created": 1794336753842, "link": null, "locked": false, "text": "**In postgres:** (database version of the lease server)\n1. FOR UPDATE -> locks the row\nthe select returns until COMMIT\n2. SKIP LOCKED -> instead of waiting\nfor the lock to release, skips it\njobs (sorted by start_tag)\n id state start_tag\n 17 Queued 100 <- replica 1 locks this\n 18 Queued 101 <- replica 2 skips 17, locks this\n 19 Queued 103 <- replica 3 skips 17 and 18, locks this", "fontSize": 19.940639845500286, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "**In postgres:** (database version of the lease server)\n1. FOR UPDATE -> locks the row\nthe select returns until COMMIT\n2. SKIP LOCKED -> instead of waiting\nfor the lock to release, skips it\njobs (sorted by start_tag)\n id state start_tag\n 17 Queued 100 <- replica 1 locks this\n 18 Queued 101 <- replica 2 skips 17, locks this\n 19 Queued 103 <- replica 3 skips 17 and 18, locks this", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "yWkauiYs3shIlAgxbMDZz", "type": "text", "x": -7001.477368543445, "y": -9361.456955231113, "width": 617.7706978319267, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9W", "roundness": null, "seed": 512347509, "version": 151, "versionNonce": 384300885, "isDeleted": false, "boundElements": [], "updated": 1794337970888, "created": 1794337764840, "link": null, "locked": false, "text": "A crashed replica loses its connection, and the database drops\nthe lock on its own.", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "A crashed replica loses its connection, and the database drops the lock on its own.", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "gzkH9QjNUgaHgJ6NIQ2Sc", "type": "text", "x": -7002.359487796915, "y": -9664.909129255302, "width": 243.79978942871094, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9X", "roundness": null, "seed": 1218321979, "version": 28, "versionNonce": 1982579483, "isDeleted": false, "boundElements": [], "updated": 1794337930983, "created": 1794337923149, "link": null, "locked": false, "text": "## Postgres as a Queue", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "## Postgres as a Queue", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "y_nmdqc3Wga3F3Af8VgQA", "type": "text", "x": -7601.599855966762, "y": -8145.02156237381, "width": 740.9246225677513, "height": 105.54406355310672, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9Y", "roundness": null, "seed": 1756059483, "version": 259, "versionNonce": 606188475, "isDeleted": false, "boundElements": [], "updated": 1794342779016, "created": 1794342313345, "link": null, "locked": false, "text": "1. What does it look like finally with the fencing_token(attempt changes from this\nchapter:\n\n\n", "fontSize": 16.887050168497076, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "1. What does it look like finally with the fencing_token(attempt changes from this chapter:\n\n\n", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "akG8ZVMCgJnMu_2lvgmMF", "type": "rectangle", "x": -7746.5374812912705, "y": -8087.324751066212, "width": 346.6979161028057, "height": 260.205220037883, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9Z", "roundness": { "type": 3 }, "seed": 972304213, "version": 257, "versionNonce": 820172277, "isDeleted": false, "boundElements": [ { "id": "WK_c_nwzGAZFjUP1RKXG0", "type": "arrow" } ], "updated": 1794342436206, "created": 1794342331760, "link": null, "locked": false }, { "id": "zbdrqFYI_CZvyASgxj7D2", "type": "text", "x": -7738.106313308037, "y": -8016.311974359513, "width": 326.19970703125, "height": 125, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9a", "roundness": null, "seed": 222062773, "version": 131, "versionNonce": 2095291573, "isDeleted": false, "boundElements": [], "updated": 1794342436206, "created": 1794342337976, "link": null, "locked": false, "text": "worker calls:\n1. heartbeat(job_id, attempt)\n2. complete(job_id, attempt, cpu)\n3. fail(job_id, attempt)\n", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "worker calls:\n1. heartbeat(job_id, attempt)\n2. complete(job_id, attempt, cpu)\n3. fail(job_id, attempt)\n", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "8D1-jriEVxBc5DA3zTuGi", "type": "text", "x": -7728.584846819874, "y": -8071.62376216318, "width": 70.27992248535156, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9b", "roundness": null, "seed": 981734811, "version": 114, "versionNonce": 135806485, "isDeleted": false, "boundElements": [], "updated": 1794342436206, "created": 1794342349006, "link": null, "locked": false, "text": "Worker:", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Worker:", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "r38W5_541oYTGt2TU9S4z", "type": "rectangle", "x": -7086.63667065016, "y": -8090.941125625801, "width": 400.6688087037591, "height": 244.85315564911707, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9d", "roundness": { "type": 3 }, "seed": 458657653, "version": 137, "versionNonce": 689407285, "isDeleted": false, "boundElements": [ { "id": "WK_c_nwzGAZFjUP1RKXG0", "type": "arrow" } ], "updated": 1794342548344, "created": 1794342401462, "link": null, "locked": false }, { "id": "5dy0u-9XCxRSHxyqpddNW", "type": "text", "x": -7070.798371395346, "y": -8068.853502454454, "width": 93.91990661621094, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9e", "roundness": null, "seed": 403590485, "version": 10, "versionNonce": 738596277, "isDeleted": false, "boundElements": [], "updated": 1794342408032, "created": 1794342405630, "link": null, "locked": false, "text": "Database", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Database", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "WK_c_nwzGAZFjUP1RKXG0", "type": "arrow", "x": -7400.836397911242, "y": -7958.240980765607, "width": 319.9536304837575, "height": 5.025118772340647, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9f", "roundness": { "type": 2 }, "seed": 478801013, "version": 182, "versionNonce": 1186666133, "isDeleted": false, "boundElements": [], "updated": 1794342548344, "created": 1794342412681, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 319.9536304837575, 5.025118772340647 ] ], "startBinding": { "elementId": "akG8ZVMCgJnMu_2lvgmMF", "mode": "inside", "fixedPoint": [ 0.9971247801717923, 0.4960844762522915 ] }, "endBinding": { "elementId": "r38W5_541oYTGt2TU9S4z", "mode": "inside", "fixedPoint": [ 0.014360746575933002, 0.5624810644870746 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "Ngjdu5rx-s01okeeNH8qg", "type": "text", "x": -7382.497399140363, "y": -7991.2880678082565, "width": 264.6797180175781, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9g", "roundness": null, "seed": 781332347, "version": 30, "versionNonce": 2026552149, "isDeleted": false, "boundElements": [], "updated": 1794342440350, "created": 1794342429296, "link": null, "locked": false, "text": "store checks attempt first", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "store checks attempt first", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Kd8cmSQl02-A7SraKoEkE", "type": "text", "x": -7073.109922214791, "y": -8034.950757102598, "width": 368.919677734375, "height": 75, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9h", "roundness": null, "seed": 1217128661, "version": 3, "versionNonce": 1204647829, "isDeleted": false, "boundElements": [], "updated": 1794342543481, "created": 1794342542792, "link": null, "locked": false, "text": "row: job 17, state Running, attempt 2\ncall attempt 2: accept, update row\ncall attempt 1: reject, stale worker", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "row: job 17, state Running, attempt 2\ncall attempt 2: accept, update row\ncall attempt 1: reject, stale worker", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "S3GMR3laVbiG1QNKZvxOq", "type": "text", "x": -7375.751895378649, "y": -7953.447192694243, "width": 214.5797576904297, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9i", "roundness": null, "seed": 213172917, "version": 28, "versionNonce": 1003366555, "isDeleted": false, "boundElements": [], "updated": 1794342557936, "created": 1794342550775, "link": null, "locked": false, "text": "inside the transaction", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "inside the transaction", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "fX0VgXXr3kKaKG7kBfKKL", "type": "text", "x": -7606.736530013436, "y": -7778.786196536522, "width": 800, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9j", "roundness": null, "seed": 1112986005, "version": 202, "versionNonce": 967260603, "isDeleted": false, "boundElements": [], "updated": 1794343455601, "created": 1794342801365, "link": null, "locked": false, "text": "2. clock problem across 2 replicas (all replicas reads the database clock, so no\nproblem of synchronization", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "2. clock problem across 2 replicas (all replicas reads the database clock, so no problem of synchronization", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "LkEzvU0il2gQ_oO12HQj_", "type": "text", "x": -7676.055607612116, "y": -7700.275052501352, "width": 767.3591918945312, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9k", "roundness": null, "seed": 1746421557, "version": 63, "versionNonce": 1610516853, "isDeleted": false, "boundElements": [], "updated": 1794343457603, "created": 1794342892375, "link": null, "locked": false, "text": "the replicas never read their own clocks for this purpose. The database\nevaluates now() on both the write and the comparison, so one clock measures\nboth ends. This is the monotonic-clock lesson from the chapter applied to the\nonly machine the replicas share.", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "the replicas never read their own clocks for this purpose. The database evaluates now() on both the write and the comparison, so one clock measures both ends. This is the monotonic-clock lesson from the chapter applied to the only machine the replicas share.", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "UZLWI0DurW1qyLWz72iOb", "type": "text", "x": -6102.3825608553625, "y": -9727.807175776548, "width": 157.69984436035156, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9l", "roundness": null, "seed": 337229147, "version": 116, "versionNonce": 1039509787, "isDeleted": false, "boundElements": [], "updated": 1794345095821, "created": 1794343264020, "link": null, "locked": false, "text": "Reaper Concept:", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Reaper Concept:", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "YwlZ3Yf5DVZ_fb6IpLMkB", "type": "text", "x": -6101.226430301683, "y": -9535.180990021703, "width": 310.6778943601593, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9m", "roundness": null, "seed": 715919963, "version": 173, "versionNonce": 1460658939, "isDeleted": false, "boundElements": [], "updated": 1794345032421, "created": 1794343281460, "link": null, "locked": false, "text": "every few seconds:", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "every few seconds:", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "LfMGRM-RxHQRWgqCNU38x", "type": "text", "x": -6103.9611166453815, "y": -9507.779609208665, "width": 296.28820094263745, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9n", "roundness": null, "seed": 865670357, "version": 101, "versionNonce": 447728539, "isDeleted": false, "boundElements": [], "updated": 1794345032421, "created": 1794343281461, "link": null, "locked": false, "text": "UPDATE jobs", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "UPDATE jobs", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "Un9fc3Vag72l8e_tCK-Ia", "type": "text", "x": -6110.862658744736, "y": -9472.779609208665, "width": 469.89959716796875, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9o", "roundness": null, "seed": 1036979963, "version": 59, "versionNonce": 2139725883, "isDeleted": false, "boundElements": [], "updated": 1794345032421, "created": 1794343281461, "link": null, "locked": false, "text": "SET state = 'Queued', attempt = attempt + 1", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "SET state = 'Queued', attempt = attempt + 1", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "LC8Qtb8zNF_nAW23Ekfjb", "type": "text", "x": -6105.694527044668, "y": -9437.779609208665, "width": 354.96155274817346, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9p", "roundness": null, "seed": 363442741, "version": 91, "versionNonce": 528474331, "isDeleted": false, "boundElements": [], "updated": 1794345032421, "created": 1794343281461, "link": null, "locked": false, "text": "WHERE state = 'Running'", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "WHERE state = 'Running'", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "v8yH1fyv4ISqYDeaJmGUq", "type": "text", "x": -6107.5126526412205, "y": -9402.779609208665, "width": 463.1995849609375, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9q", "roundness": null, "seed": 950630299, "version": 59, "versionNonce": 874843515, "isDeleted": false, "boundElements": [], "updated": 1794345032421, "created": 1794343281461, "link": null, "locked": false, "text": "AND last_heartbeat < now() - lease_timeout", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "AND last_heartbeat < now() - lease_timeout", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "2d5nOnFdcpLpceQSES2S2", "type": "text", "x": -6104.928732147855, "y": -9367.779609208665, "width": 389.2657346580018, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9r", "roundness": null, "seed": 444572565, "version": 74, "versionNonce": 549168667, "isDeleted": false, "boundElements": [], "updated": 1794345032421, "created": 1794343281461, "link": null, "locked": false, "text": "AND attempt < max_attempts", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "AND attempt < max_attempts", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "nBqBcb_TzHCtVhUlZnFxB", "type": "text", "x": -5934.922884088829, "y": -9730.704091219211, "width": 800, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9t", "roundness": null, "seed": 1519011803, "version": 114, "versionNonce": 1998471899, "isDeleted": false, "boundElements": [], "updated": 1794345100427, "created": 1794343330850, "link": null, "locked": false, "text": "small background loop that find leases that nobody renewed, and put those jobs\nback in the queue.", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "small background loop that find leases that nobody renewed, and put those jobs back in the queue.", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "PjIBCW4EVcfaKDGJ6U5yG", "type": "text", "x": -7876.16246267387, "y": -10499.197948824507, "width": 1094.3615090033227, "height": 88.00564114591366, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9u", "roundness": null, "seed": 828811131, "version": 251, "versionNonce": 158193699, "isDeleted": false, "boundElements": [], "updated": 1794636639528, "created": 1794343368560, "link": null, "locked": false, "text": "Lease and Fencing Mechanism", "fontSize": 70.40451291673092, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Lease and Fencing Mechanism", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "MKdtk9huU5oWlP2uXB8wB", "type": "text", "x": -6110.011753375698, "y": -9663.552802366581, "width": 800, "height": 125.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9w", "roundness": null, "seed": 1809783765, "version": 160, "versionNonce": 1070867290, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794345034034, "link": null, "locked": false, "text": "- Idempotent\n- mints the fencing_token (.i.e this is responsible for increasing the fencing\ntoken)\n- uses the database clock now(): one shared clock, but a wall clock,\nNOT monotonic (v5)", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "- Idempotent\n- mints the fencing_token (.i.e this is responsible for increasing the fencing\ntoken)\n- uses the database clock now(): one shared clock, but a wall clock,\nNOT monotonic (v5)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "h_sremJFCXpZ3xGf4ekuQ", "type": "text", "x": -7429.689560466204, "y": -7501.56932211383, "width": 173.17982482910156, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9y", "roundness": null, "seed": 34714389, "version": 47, "versionNonce": 2057248859, "isDeleted": false, "boundElements": [], "updated": 1794383781285, "created": 1794382080309, "link": null, "locked": false, "text": "Final Architecture", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Final Architecture", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "33MBuybf4Xbr5_RwJbijq", "type": "rectangle", "x": -7619.28744461026, "y": -7329.625613312317, "width": 409.58982770098424, "height": 198.5424244202486, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "b9z", "roundness": { "type": 3 }, "seed": 647222613, "version": 217, "versionNonce": 1696603477, "isDeleted": false, "boundElements": [ { "id": "mqiP8xh0wTjPpj1RwdA2Q", "type": "arrow" } ], "updated": 1794383046913, "created": 1794382095363, "link": null, "locked": false }, { "id": "qAtpj0QS4tVOeHnPlKH8C", "type": "rectangle", "x": -7855.846339841081, "y": -7326.91512839786, "width": 205.31861809502425, "height": 147.04335993649784, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bA1", "roundness": { "type": 3 }, "seed": 1453952661, "version": 69, "versionNonce": 868423675, "isDeleted": false, "boundElements": [ { "id": "LokRnp63ffTtOpAC-qO_t", "type": "arrow" }, { "id": "n6lVRMerVrCdXp7ecqsub", "type": "arrow" } ], "updated": 1794383275095, "created": 1794382106802, "link": null, "locked": false }, { "id": "twwq1mEmrUxx8vwmtoLcZ", "type": "rectangle", "x": -7044.473557288555, "y": -7346.9360549331495, "width": 231.5306854752606, "height": 220.0581636020187, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bA4", "roundness": { "type": 3 }, "seed": 1830432373, "version": 176, "versionNonce": 1435905531, "isDeleted": false, "boundElements": [ { "id": "mqiP8xh0wTjPpj1RwdA2Q", "type": "arrow" }, { "id": "1dlalX_16kTissWrUVNAp", "type": "arrow" } ], "updated": 1794384114695, "created": 1794382119673, "link": null, "locked": false }, { "id": "aCq5Yqq6cSp8Xlzl-zL7r", "type": "text", "x": -7500.09629288922, "y": -7313.801142735347, "width": 105.05989074707031, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bA5", "roundness": null, "seed": 107784437, "version": 12, "versionNonce": 80530453, "isDeleted": false, "boundElements": [], "updated": 1794382132064, "created": 1794382129246, "link": null, "locked": false, "text": "Dispatcher", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Dispatcher", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "hvqZ86ndOxz4-aJO-V74-", "type": "text", "x": -7842.7103367404625, "y": -7314.043207546129, "width": 93.91990661621094, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bA6", "roundness": null, "seed": 923822133, "version": 14, "versionNonce": 21119029, "isDeleted": false, "boundElements": [], "updated": 1794382205195, "created": 1794382134824, "link": null, "locked": false, "text": "Database", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Database", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "ac9pCMe4psekHQEjeDsuA", "type": "text", "x": -7011.606271626967, "y": -7334.372251727765, "width": 64.99992370605469, "height": 25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bA8", "roundness": null, "seed": 820989845, "version": 60, "versionNonce": 727598235, "isDeleted": false, "boundElements": [], "updated": 1794383030732, "created": 1794382142086, "link": null, "locked": false, "text": "Worker", "fontSize": 20, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Worker", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "7qx4ZfUHoSxaICeAMxiaz", "type": "text", "x": -7847.828876769424, "y": -7282.788091493002, "width": 128.4802703857422, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bA9", "roundness": null, "seed": 198760117, "version": 34, "versionNonce": 1489751189, "isDeleted": false, "boundElements": [], "updated": 1794382206813, "created": 1794382190509, "link": null, "locked": false, "text": "transaction with\nUPDATE SKIP \nLOCKED ", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "transaction with\nUPDATE SKIP \nLOCKED ", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "niPXl4-bEDvpfoYNZtpGL", "type": "line", "x": -7468.404054248847, "y": -7129.527941617795, "width": 0.9776228265818645, "height": 62.5694048836458, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAA", "roundness": { "type": 2 }, "seed": 1505310901, "version": 42, "versionNonce": 1241538485, "isDeleted": false, "boundElements": [], "updated": 1794382241524, "created": 1794382240668, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0.9776228265818645, 62.5694048836458 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "5tngY5OQEDKhN3LyXvP2V", "type": "rectangle", "x": -7582.637236997869, "y": -7065.003264229117, "width": 474.00708293635944, "height": 320.00293490562893, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAB", "roundness": { "type": 3 }, "seed": 444258139, "version": 358, "versionNonce": 1335145531, "isDeleted": false, "boundElements": [], "updated": 1794384100749, "created": 1794382246629, "link": null, "locked": false }, { "id": "ME-aQrlG2_BcB81oUd8rC", "type": "text", "x": -7572.488551974242, "y": -7050.419239542806, "width": 445.9460625118837, "height": 301.88901127591635, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAC", "roundness": null, "seed": 27639605, "version": 965, "versionNonce": 1055558989, "isDeleted": false, "boundElements": [], "updated": 1794636635191, "created": 1794382252250, "link": null, "locked": false, "text": "1. dispatcher crashes, while db mid transaction #rollback\n2. dispatcher crashes after transaction commit before\njob reply to worker, i.e job is running but no worker\nknows then, waits for lease_timeout then #Reaper \nexpires the lease and job goes back to Queue\n3. dispatcher pauses mid transaction, holds the row lock\nwhile asleep\n4. dispatcher pauses after commit but before reply:\nREAPER handles it with lease_timeout + late reply is \nhandled by fencing\n5. Network delay -> reply/job reaches the worker late ?\nlease_timeout + fencing solves this !!!\n6. Reapers replica is down -> run reaper on replica,\nsince idempotent 2 replicas running simultaenously does\nno harm !", "fontSize": 16.10074726804887, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "1. dispatcher crashes, while db mid transaction #rollback\n2. dispatcher crashes after transaction commit before job reply to worker, i.e job is running but no worker knows then, waits for lease_timeout then #Reaper \nexpires the lease and job goes back to Queue\n3. dispatcher pauses mid transaction, holds the row lock\nwhile asleep\n4. dispatcher pauses after commit but before reply: REAPER handles it with lease_timeout + late reply is \nhandled by fencing\n5. Network delay -> reply/job reaches the worker late ?\nlease_timeout + fencing solves this !!!\n6. Reapers replica is down -> run reaper on replica,\nsince idempotent 2 replicas running simultaenously does\nno harm !", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "bxOIjYGythGisyCxMFdo7", "type": "text", "x": -7608.200385602555, "y": -10291.537818687799, "width": 205.12037658691406, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAD", "roundness": null, "seed": 507997941, "version": 27, "versionNonce": 1148888757, "isDeleted": false, "boundElements": [], "updated": 1794382609233, "created": 1794382600986, "link": null, "locked": false, "text": "heartbeat===lease renewal", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "heartbeat===lease renewal", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "3uRdH0bWcy4e_V76X_OHV", "type": "text", "x": -7439.001615668591, "y": -7434.252818045218, "width": 208.4804229736328, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAH", "roundness": null, "seed": 56258907, "version": 140, "versionNonce": 1786103227, "isDeleted": false, "boundElements": [], "updated": 1794383984703, "created": 1794382662178, "link": null, "locked": false, "text": "heartbeat(job_id,attempt)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "heartbeat(job_id,attempt)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "rjNKrMXrhyhhMqARjjQOI", "type": "rectangle", "x": -7611.522666309754, "y": -7255.85272865852, "width": 392.35520240349143, "height": 117.45665906950946, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAI", "roundness": { "type": 3 }, "seed": 360165659, "version": 249, "versionNonce": 2000744763, "isDeleted": false, "boundElements": [ { "id": "LokRnp63ffTtOpAC-qO_t", "type": "arrow" } ], "updated": 1794383115161, "created": 1794382761460, "link": null, "locked": false }, { "id": "Ola3XQTCmGD-NLJ7aBzgj", "type": "text", "x": -7597.152978150288, "y": -7250.483447620751, "width": 369.67809053159596, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAJ", "roundness": null, "seed": 1131137339, "version": 358, "versionNonce": 1272300411, "isDeleted": false, "boundElements": [], "updated": 1794382984185, "created": 1794382766907, "link": null, "locked": false, "text": "Reaper: background worker finds RUNNING job\nwhose last heartbeat is older than\nlease_timeout, emits \nLeaseExpired on that job and goes back to\nQueue with +1 fencing_token(attempt)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Reaper: background worker finds RUNNING job whose last heartbeat is older than lease_timeout, emits \nLeaseExpired on that job and goes back to Queue with +1 fencing_token(attempt)", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "V7BBpoPYoGa4Bf6lVLW5K", "type": "text", "x": -7837.564228718029, "y": -7204.125517530027, "width": 136.416259765625, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAK", "roundness": null, "seed": 649446235, "version": 18, "versionNonce": 398313979, "isDeleted": false, "boundElements": [], "updated": 1794382788287, "created": 1794382783653, "link": null, "locked": false, "text": "heartbeat stored", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "heartbeat stored", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "mqiP8xh0wTjPpj1RwdA2Q", "type": "arrow", "x": -7049.973557288555, "y": -7271.636959655794, "width": 154.22405962072025, "height": 0.2380574806511504, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAL", "roundness": { "type": 2 }, "seed": 531055957, "version": 76, "versionNonce": 1306405755, "isDeleted": false, "boundElements": [], "updated": 1794383860552, "created": 1794383044971, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -154.22405962072025, -0.2380574806511504 ] ], "startBinding": { "elementId": "twwq1mEmrUxx8vwmtoLcZ", "mode": "orbit", "fixedPoint": [ 0.34277341211317236, 0.34277341211316725 ] }, "endBinding": { "elementId": "33MBuybf4Xbr5_RwJbijq", "mode": "orbit", "fixedPoint": [ 0.7100931087136029, 0.2899068912864107 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "J9DAbSRi4q5G26pNQ2nRX", "type": "text", "x": -7409.260046001828, "y": -7412.772918919121, "width": 120.960205078125, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAN", "roundness": null, "seed": 1326407899, "version": 56, "versionNonce": 1415884053, "isDeleted": false, "boundElements": [], "updated": 1794383986291, "created": 1794383065280, "link": null, "locked": false, "text": "renew the lease", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "renew the lease", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "-hn-srh2Lxc81neXBLAay", "type": "text", "x": -7167.686543095102, "y": -7305.025780349907, "width": 104.3201904296875, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAO", "roundness": null, "seed": 1800634011, "version": 43, "versionNonce": 493494197, "isDeleted": false, "boundElements": [], "updated": 1794383091365, "created": 1794383078241, "link": null, "locked": false, "text": "lease(size)\n\nget the lease", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "lease(size)\n\nget the lease", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "LokRnp63ffTtOpAC-qO_t", "type": "arrow", "x": -7608.3987927067155, "y": -7190.251932108456, "width": 43.10909879799783, "height": 14.369688159464204, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAP", "roundness": { "type": 2 }, "seed": 1811748699, "version": 43, "versionNonce": 372671899, "isDeleted": false, "boundElements": [], "updated": 1794383117208, "created": 1794383115158, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -43.10909879799783, -14.369688159464204 ] ], "startBinding": { "elementId": "rjNKrMXrhyhhMqARjjQOI", "mode": "inside", "fixedPoint": [ 0.007961850853262429, 0.5585106631650665 ] }, "endBinding": { "elementId": "qAtpj0QS4tVOeHnPlKH8C", "mode": "inside", "fixedPoint": [ 0.9952261038587223, 0.8316833088060138 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "n6lVRMerVrCdXp7ecqsub", "type": "arrow", "x": -7682.355003024024, "y": -7174.373113322417, "width": 148.46173352818735, "height": 65.96598574769541, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAQ", "roundness": { "type": 2 }, "seed": 1060750875, "version": 245, "versionNonce": 872956597, "isDeleted": false, "boundElements": [], "updated": 1794383297217, "created": 1794383275092, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -71.39213488917358, 65.96598574769541 ], [ -148.46173352818735, 2.3697155987236336 ] ], "startBinding": { "elementId": "qAtpj0QS4tVOeHnPlKH8C", "mode": "orbit", "fixedPoint": [ 0.9289966071766317, 0.928996607176632 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "6TRHBLTs3gkw1fRZOESFd", "type": "text", "x": -7867.6781396235465, "y": -7106.286660669875, "width": 223.0564422607422, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAS", "roundness": null, "seed": 2059887707, "version": 115, "versionNonce": 1407136501, "isDeleted": false, "boundElements": [], "updated": 1794384078793, "created": 1794383298705, "link": null, "locked": false, "text": "transaction_timeout on row,\nto kill stuck transaction and\nroll it back !", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "transaction_timeout on row,\nto kill stuck transaction and\nroll it back !", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "yacM-ggPj4OSZD7y4U4cm", "type": "text", "x": -7038.155478072188, "y": -7309.964396646456, "width": 213.78340445945005, "height": 179.99999999999997, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAT", "roundness": null, "seed": 435583733, "version": 172, "versionNonce": 916078747, "isDeleted": false, "boundElements": [], "updated": 1794417815173, "created": 1794383847976, "link": null, "locked": false, "text": ". Worker must treat a\nrejected heartbeat as\n\u201cstop now, kill the\ncontainer.\u201d That is the\nattempt token doing the\nfencing job, and it is why\nthe heartbeat must carry\n\"attempt\", not only\ncomplete.", "fontSize": 15.999999999999996, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": ". Worker must treat a rejected heartbeat as \u201cstop now, kill the container.\u201d That is the attempt token doing the fencing job, and it is why the heartbeat must carry \"attempt\", not only complete.", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "YGsZiR1bzbPKQtWmEB91n", "type": "text", "x": -7447.913321291539, "y": -6661.696766959884, "width": 214.88043212890625, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAU", "roundness": null, "seed": 90006293, "version": 97, "versionNonce": 1065154549, "isDeleted": false, "boundElements": [], "updated": 1794384128718, "created": 1794384000111, "link": null, "locked": false, "text": "heartbeat(job_id, attempt)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "heartbeat(job_id, attempt)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "TF2pVji5e2MDu6DSIV76K", "type": "text", "x": -7503.654412647936, "y": -6695.184804046009, "width": 311.4245910644531, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAV", "roundness": null, "seed": 334349499, "version": 88, "versionNonce": 850677973, "isDeleted": false, "boundElements": [], "updated": 1794384123179, "created": 1794384000111, "link": null, "locked": false, "text": "complete(job_id, attempt, cpu_seconds)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "complete(job_id, attempt, cpu_seconds)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "07a_oAQX58ON3NwTGP1-K", "type": "text", "x": -7419.836431838587, "y": -6627.838787247674, "width": 161.21632385253906, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAW", "roundness": null, "seed": 1045051509, "version": 75, "versionNonce": 1843944597, "isDeleted": false, "boundElements": [], "updated": 1794384126216, "created": 1794384000111, "link": null, "locked": false, "text": "fail(job_id, attempt)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "fail(job_id, attempt)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "1dlalX_16kTissWrUVNAp", "type": "arrow", "x": -6910.442960838301, "y": -7121.377891331131, "width": 897.838535117121, "height": 541.7726173123392, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAZ", "roundness": { "type": 2 }, "seed": 607074843, "version": 715, "versionNonce": 1017844059, "isDeleted": false, "boundElements": [], "updated": 1794384114695, "created": 1794384102934, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -420.4446844769964, 541.7726173123392 ], [ -897.838535117121, 111.12434137777109 ] ], "startBinding": { "elementId": "twwq1mEmrUxx8vwmtoLcZ", "mode": "orbit", "fixedPoint": [ 0.7682574707150847, 0.7682574707150875 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "Xy15WCkvP7eFhMSJ-NpzL", "type": "text", "x": -7609.668100208923, "y": -10309.712541780977, "width": 285.45654296875, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAa", "roundness": null, "seed": 1315907675, "version": 53, "versionNonce": 1050599643, "isDeleted": false, "boundElements": [], "updated": 1794384199879, "created": 1794384187492, "link": null, "locked": false, "text": "database === lease server + storage", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "database === lease server + storage", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "3Z_y7HwQQ5ZlXKiNsg3G5", "type": "text", "x": -8244.21933471762, "y": -7363.646804751416, "width": 206.73638916015625, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAd", "roundness": null, "seed": 915624405, "version": 50, "versionNonce": 10175925, "isDeleted": false, "boundElements": [], "updated": 1794384441512, "created": 1794384331418, "link": null, "locked": false, "text": "scheduler_state (one row)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "scheduler_state (one row)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "JNcvW79YxGuZ-JRo8oy1O", "type": "text", "x": -8213.061266299645, "y": -7331.004194611704, "width": 203.62801728580968, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAe", "roundness": null, "seed": 744694267, "version": 83, "versionNonce": 597035771, "isDeleted": false, "boundElements": [], "updated": 1794384408418, "created": 1794384331418, "link": null, "locked": false, "text": "id virtual_now", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "id virtual_now", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "PXUbOy3oZqCArX7cxDRkW", "type": "text", "x": -8210.938471997957, "y": -7301.004194611704, "width": 201.50098486166556, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAf", "roundness": null, "seed": 408178485, "version": 92, "versionNonce": 672658939, "isDeleted": false, "boundElements": [], "updated": 1794384405235, "created": 1794384331418, "link": null, "locked": false, "text": "1 4815.25", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "1 4815.25", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "XvK3rrV9vek-g0YenqPCy", "type": "text", "x": -8340.669869518026, "y": -7630.426802326677, "width": 59.568115234375, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAg", "roundness": null, "seed": 743428027, "version": 159, "versionNonce": 1812625115, "isDeleted": false, "boundElements": [], "updated": 1794417674503, "created": 1794384382281, "link": null, "locked": false, "text": "BEGIN;", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "BEGIN;", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "1oOvKAHIwJz14a0vVDRqE", "type": "text", "x": -8344.803360187048, "y": -7600.426802326677, "width": 605.00927734375, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAh", "roundness": null, "seed": 1562533237, "version": 107, "versionNonce": 819572469, "isDeleted": false, "boundElements": [], "updated": 1794384386157, "created": 1794384382281, "link": null, "locked": false, "text": "SELECT virtual_now FROM scheduler_state FOR UPDATE; -- lock the row", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "SELECT virtual_now FROM scheduler_state FOR UPDATE; -- lock the row", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "dIxpfseiD4-IPVO1CMDCI", "type": "text", "x": -8344.222084487586, "y": -7570.426802326677, "width": 487.2357409021003, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAi", "roundness": null, "seed": 1121198171, "version": 152, "versionNonce": 1989601371, "isDeleted": false, "boundElements": [], "updated": 1794384495705, "created": 1794384382281, "link": null, "locked": false, "text": "SELECT ... FROM jobs WHERE state = 'queued'", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "SELECT ... FROM jobs WHERE state = 'queued'", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "D2XubCiX37Mj901VI5dsG", "type": "text", "x": -8347.195388995642, "y": -7540.426802326677, "width": 609.7933349609375, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAj", "roundness": null, "seed": 474015445, "version": 107, "versionNonce": 1303918005, "isDeleted": false, "boundElements": [], "updated": 1794384386157, "created": 1794384382281, "link": null, "locked": false, "text": "ORDER BY start_tag FOR UPDATE SKIP LOCKED LIMIT 1; -- pick the job", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "ORDER BY start_tag FOR UPDATE SKIP LOCKED LIMIT 1; -- pick the job", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "NzsTscqWNlQtS0RFOIrwy", "type": "text", "x": -8344.143870358173, "y": -7510.426802326677, "width": 502.83754996604654, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAk", "roundness": null, "seed": 1908759803, "version": 154, "versionNonce": 723064053, "isDeleted": false, "boundElements": [], "updated": 1794384501511, "created": 1794384382281, "link": null, "locked": false, "text": "UPDATE jobs SET state = 'running', attempt = ... ;", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "UPDATE jobs SET state = 'running', attempt = ... ;", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "wQLSTNqcaEKL2PlGtROKQ", "type": "text", "x": -8343.536991707953, "y": -7480.426802326677, "width": 581.9908336693424, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAl", "roundness": null, "seed": 2106016821, "version": 125, "versionNonce": 1888130235, "isDeleted": false, "boundElements": [], "updated": 1794384503862, "created": 1794384382281, "link": null, "locked": false, "text": "UPDATE scheduler_state SET virtual_now = ;", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "UPDATE scheduler_state SET virtual_now = ;", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "hMiVKZzZrd12Om60rMyb-", "type": "text", "x": -8343.010727505603, "y": -7453.053198657848, "width": 74.04814147949219, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAm", "roundness": null, "seed": 1957037467, "version": 161, "versionNonce": 41478075, "isDeleted": false, "boundElements": [], "updated": 1794384506865, "created": 1794384382281, "link": null, "locked": false, "text": "COMMIT;", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "COMMIT;", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "p7DOHtRMIdd1fPm6fwbZv", "type": "rectangle", "x": -8367.590868670502, "y": -7649.226156703509, "width": 663.2931490835908, "height": 231.22769770301238, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAn", "roundness": { "type": 3 }, "seed": 246655477, "version": 112, "versionNonce": 1497992475, "isDeleted": false, "boundElements": [], "updated": 1794384713114, "created": 1794384393006, "link": null, "locked": false }, { "id": "GFXpNBe5Gl-N1L_ZpPCRp", "type": "line", "x": -8242.067249940388, "y": -7303.045244538611, "width": 184.98212912991585, "height": 6.606489058660372, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAq", "roundness": { "type": 2 }, "seed": 2139086203, "version": 51, "versionNonce": 71906331, "isDeleted": false, "boundElements": [], "updated": 1794384419712, "created": 1794384418613, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 184.98212912991585, -6.606489058660372 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "xTHSFNmLnNhZoOIadiiRl", "type": "line", "x": -8191.857802448342, "y": -7332.1138834942, "width": 1.3213413604726156, "height": 66.06503574907947, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAr", "roundness": { "type": 2 }, "seed": 592244181, "version": 45, "versionNonce": 1363284245, "isDeleted": false, "boundElements": [], "updated": 1794384437984, "created": 1794384424870, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -1.3213413604726156, 66.06503574907947 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "uyr_PaCaWqaVS2p6AV0rl", "type": "line", "x": -8255.280300638944, "y": -7340.041713913334, "width": 229.9064289237731, "height": 2.642610139710996, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAs", "roundness": { "type": 2 }, "seed": 1775280315, "version": 54, "versionNonce": 1475012923, "isDeleted": false, "boundElements": [], "updated": 1794384447472, "created": 1794384445652, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 229.9064289237731, -2.642610139710996 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "TKVWY_UjRyPXRCuFWN-w7", "type": "rectangle", "x": -8283.02759823406, "y": -7377.038110706823, "width": 298.61407481256356, "height": 128.16619257920865, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAw", "roundness": { "type": 3 }, "seed": 1290653365, "version": 61, "versionNonce": 627421659, "isDeleted": false, "boundElements": [], "updated": 1794384465840, "created": 1794384463908, "link": null, "locked": false }, { "id": "3DStmUgOVGXLTl7ethLgZ", "type": "line", "x": -7857.56867220276, "y": -7288.510925060816, "width": 122.88097229978666, "height": 1.3213413604726156, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAx", "roundness": { "type": 2 }, "seed": 1940175541, "version": 33, "versionNonce": 937400661, "isDeleted": false, "boundElements": [], "updated": 1794384471637, "created": 1794384471036, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -122.88097229978666, -1.3213413604726156 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "-WqG1VcDUY5qQ0MSJ_7cm", "type": "line", "x": -8144.290965096007, "y": -7415.355848860786, "width": 1.3212687792383804, "height": 34.353786653778116, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bAz", "roundness": { "type": 2 }, "seed": 1608366651, "version": 13, "versionNonce": 63436059, "isDeleted": false, "boundElements": [], "updated": 1794384474124, "created": 1794384473884, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1.3212687792383804, 34.353786653778116 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "jBzWhmObgcMxhhnWlOGWd", "type": "text", "x": -8286.016483639905, "y": -7189.363337952132, "width": 275.64286213795276, "height": 160, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB0", "roundness": null, "seed": 491134363, "version": 165, "versionNonce": 340123835, "isDeleted": false, "boundElements": [], "updated": 1794384681904, "created": 1794384675580, "link": null, "locked": false, "text": "The FOR UPDATE on that single\nrow means two replicas cannot\ndispatch at the same time. One\nwaits for the other to commit.\nThat is the mutual exclusion a\nleader lease would otherwise give\nyou, and it is why no leader lease\nis needed.", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "The FOR UPDATE on that single row means two replicas cannot dispatch at the same time. One waits for the other to commit. That is the mutual exclusion a leader lease would otherwise give you, and it is why no leader lease is needed.", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "5E-NC6jc2bneBqloAeUAq", "type": "rectangle", "x": -8303.801973245714, "y": -7199.284227477145, "width": 313.06377077613615, "height": 180.1692359620638, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB1", "roundness": { "type": 3 }, "seed": 640267931, "version": 94, "versionNonce": 665103355, "isDeleted": false, "boundElements": [], "updated": 1794384687460, "created": 1794384685391, "link": null, "locked": false }, { "id": "tD3n2xCdj3QbSPGT7L7lv", "type": "line", "x": -8163.5535979544575, "y": -7247.084253296671, "width": 0, "height": 48.85054471753574, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB2", "roundness": { "type": 2 }, "seed": 1670207099, "version": 42, "versionNonce": 27737979, "isDeleted": false, "boundElements": [], "updated": 1794384704955, "created": 1794384702977, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 48.85054471753574 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "O4gOj7lgXfysi1MXfDwku", "type": "text", "x": -20719.566360492525, "y": -11085.284171669127, "width": 88.54417419433594, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB3", "roundness": null, "seed": 990014467, "version": 53, "versionNonce": 1940773133, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794554538577, "link": null, "locked": false, "text": "Consumer A", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Consumer A", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "KHswlf4HzPDuLIHgHudri", "type": "rectangle", "x": -20757.877966014796, "y": -11056.30249789314, "width": 178.5714444247169, "height": 287.6623396439991, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB4", "roundness": { "type": 3 }, "seed": 1583991395, "version": 67, "versionNonce": 1890104547, "isDeleted": true, "boundElements": null, "updated": 1794554588239, "created": 1794554580888, "link": null, "locked": false }, { "id": "0LokYiKeYbZTSzkLSwWhh", "type": "rectangle", "x": -20754.63122862133, "y": -11055.653139317146, "width": 103.2467928799706, "height": 23.376617431640625, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB5", "roundness": { "type": 3 }, "seed": 468628973, "version": 11, "versionNonce": 901896813, "isDeleted": true, "boundElements": null, "updated": 1794554591820, "created": 1794554589893, "link": null, "locked": false }, { "id": "_Su2wbIKsylOz13xO_gPv", "type": "rectangle", "x": -20589.696147832976, "y": -10852.406388052053, "width": 2.5973788174705987, "height": 4.545454545454049, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB6", "roundness": { "type": 3 }, "seed": 139494691, "version": 6, "versionNonce": 1242520483, "isDeleted": true, "boundElements": null, "updated": 1794554591213, "created": 1794554590625, "link": null, "locked": false }, { "id": "BErxsFOY2OQKvWDfd4NrE", "type": "rectangle", "x": -20783.202673046046, "y": -11054.354449908407, "width": 217.53245960582353, "height": 303.8961098410855, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB7", "roundness": { "type": 3 }, "seed": 1293105795, "version": 94, "versionNonce": 1233242691, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794554593239, "link": null, "locked": false }, { "id": "eNmlYulsps-ndLU-vRHrh", "type": "rectangle", "x": -20755.280587197325, "y": -11033.575225165867, "width": 42.85716663707353, "height": 9.090909090908099, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB8", "roundness": { "type": 3 }, "seed": 297863747, "version": 7, "versionNonce": 17389059, "isDeleted": true, "boundElements": null, "updated": 1794554611291, "created": 1794554609289, "link": null, "locked": false }, { "id": "mAkgzWhKT821j9H7Zw0dK", "type": "rectangle", "x": -20686.449410439513, "y": -11017.990813544773, "width": 18.181818181819835, "height": 3.24676513671875, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bB9", "roundness": { "type": 3 }, "seed": 1656246509, "version": 10, "versionNonce": 839014125, "isDeleted": true, "boundElements": null, "updated": 1794554610870, "created": 1794554609399, "link": null, "locked": false }, { "id": "0WB3OFMeWO4qpWnPIRmsC", "type": "rectangle", "x": -20667.6182336817, "y": -11014.094703703688, "width": 8.441550514915434, "height": 24.02597600763511, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBA", "roundness": { "type": 3 }, "seed": 1457781549, "version": 15, "versionNonce": 1862780707, "isDeleted": true, "boundElements": null, "updated": 1794554610548, "created": 1794554609606, "link": null, "locked": false }, { "id": "GyZZ0vzTueOROztObQ64a", "type": "rectangle", "x": -20767.6182336817, "y": -11036.172631726591, "width": 187.01305042613606, "height": 101.29870328036122, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBB", "roundness": { "type": 3 }, "seed": 1775682061, "version": 108, "versionNonce": 1611214701, "isDeleted": true, "boundElements": [ { "id": "oZjs-1lCT4ugLURRQxWV2", "type": "arrow" }, { "id": "wwJ3mgQ0brHiSaM3uGpgX", "type": "arrow" } ], "updated": 1794636714099, "created": 1794554612590, "link": null, "locked": false }, { "id": "T6hEPwG17UqECnt0RNDKA", "type": "rectangle", "x": -20583.851976135535, "y": -10935.52325927897, "width": 7.792191938919132, "height": 0.6493308327408158, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBC", "roundness": { "type": 3 }, "seed": 535675427, "version": 11, "versionNonce": 390911949, "isDeleted": true, "boundElements": null, "updated": 1794554618324, "created": 1794554615158, "link": null, "locked": false }, { "id": "JHXhaGRwzD-zcS3lRKMgU", "type": "text", "x": -20755.54031953042, "y": -11033.055746628048, "width": 120.40022277832031, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBD", "roundness": null, "seed": 917477677, "version": 30, "versionNonce": 1677036003, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794554624293, "link": null, "locked": false, "text": "Main thread\nListener thread", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Main thread\nListener thread", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "g9jpKfrm5llOldr85q6ol", "type": "rectangle", "x": -21197.488312028632, "y": -11075.133660779326, "width": 200.64938631924815, "height": 324.6753484552555, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBE", "roundness": { "type": 3 }, "seed": 437385955, "version": 110, "versionNonce": 661305805, "isDeleted": true, "boundElements": [ { "id": "wwJ3mgQ0brHiSaM3uGpgX", "type": "arrow" } ], "updated": 1794636714099, "created": 1794554676277, "link": null, "locked": false }, { "id": "nwA9_98btrK6ZUJJz5vuK", "type": "rectangle", "x": -20981.254541831546, "y": -10752.4063603088, "width": 1.9480202414779342, "height": 0.6493308327426348, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBF", "roundness": { "type": 3 }, "seed": 939167235, "version": 5, "versionNonce": 699091629, "isDeleted": true, "boundElements": null, "updated": 1794554680234, "created": 1794554677990, "link": null, "locked": false }, { "id": "pj4cKonKAUqTYB0fH8znB", "type": "text", "x": -21185.15069328751, "y": -11068.64015824914, "width": 43.920074462890625, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBG", "roundness": null, "seed": 1706857389, "version": 8, "versionNonce": 1695569283, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794554685316, "link": null, "locked": false, "text": "Kafka", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Kafka", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "oZjs-1lCT4ugLURRQxWV2", "type": "arrow", "x": -20772.29114569955, "y": -11019.748261884535, "width": 225.19719407233424, "height": 4.3548687721158785, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBH", "roundness": { "type": 2 }, "seed": 1183731565, "version": 106, "versionNonce": 1928067117, "isDeleted": true, "boundElements": [], "updated": 1794636714099, "created": 1794555506241, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -225.19719407233424, 4.3548687721158785 ] ], "startBinding": { "elementId": "GyZZ0vzTueOROztObQ64a", "mode": "orbit", "fixedPoint": [ 0.15568713842277931, 0.1556871384227701 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "SHlAFd4YlcpxFpMgCPnLW", "type": "text", "x": -20888.089747313355, "y": -11027.570827498477, "width": 6.4000091552734375, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBHV", "roundness": null, "seed": 1883136323, "version": 3, "versionNonce": 764111341, "isDeleted": true, "boundElements": null, "updated": 1794555551481, "created": 1794555551294, "link": null, "locked": false, "text": "", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "oZjs-1lCT4ugLURRQxWV2", "originalText": "", "autoResize": true, "lineHeight": 1.25, "labelPosition": 0.5 }, { "id": "vrihSkPEWpBUO3o0N_AmD", "type": "text", "x": -20962.942885226432, "y": -11041.627177181137, "width": 104.32759094238281, "height": 18.701296719636957, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBI", "roundness": null, "seed": 1914533219, "version": 29, "versionNonce": 2081467683, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555511039, "link": null, "locked": false, "text": "consumer.poll()", "fontSize": 14.961037375709566, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "consumer.poll()", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "wwJ3mgQ0brHiSaM3uGpgX", "type": "arrow", "x": -20991.338925709384, "y": -10978.073641926008, "width": 218.22069202768398, "height": 7.2566137387293566, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBJ", "roundness": { "type": 2 }, "seed": 1137922819, "version": 82, "versionNonce": 209087117, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555555099, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 218.22069202768398, -7.2566137387293566 ] ], "startBinding": { "elementId": "g9jpKfrm5llOldr85q6ol", "mode": "orbit", "fixedPoint": [ 0.6942076011013962, 0.3057923988986081 ] }, "endBinding": { "elementId": "GyZZ0vzTueOROztObQ64a", "mode": "orbit", "fixedPoint": [ 0, 0.5001 ] }, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "6oJMnxzPRY9Wg9mD9ARwi", "type": "text", "x": -20967.87796601478, "y": -11001.367430976414, "width": 57.48809814453125, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBK", "roundness": null, "seed": 535778083, "version": 10, "versionNonce": 1818562755, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555574419, "link": null, "locked": false, "text": "records", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "records", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "UkIBRghcPj0XsnbhVK7X_", "type": "text", "x": -20501.32060406521, "y": -11040.71807240042, "width": 207.01517001065437, "height": 80, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBL", "roundness": null, "seed": 1311028909, "version": 209, "versionNonce": 1153667309, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555617324, "link": null, "locked": false, "text": "Avoid long-running\nprocessing directly in the\nlistener thread, as it can\ncause consumer timeouts.", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Avoid long-running processing directly in the listener thread, as it can cause consumer timeouts.", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "RlKb_-X2-tyb7i3uqjg5_", "type": "line", "x": -20582.55325898353, "y": -10992.666120385149, "width": 72.72727272727207, "height": 0.6493447043685592, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBM", "roundness": { "type": 2 }, "seed": 1398866595, "version": 47, "versionNonce": 128712803, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555647760, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 72.72727272727207, 0.6493447043685592 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "KwdGPHnmEocYkELo7Lebu", "type": "diamond", "x": -20516.319461043193, "y": -11051.10768477169, "width": 50.64930308948897, "height": 116.23375632546231, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBN", "roundness": { "type": 2 }, "seed": 494745411, "version": 52, "versionNonce": 383549731, "isDeleted": true, "boundElements": null, "updated": 1794555654755, "created": 1794555653024, "link": null, "locked": false }, { "id": "ipgFpVJG_UzEFbgCfeIwo", "type": "rectangle", "x": -20511.774006497737, "y": -11049.808981491327, "width": 221.4285555752831, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBO", "roundness": { "type": 3 }, "seed": 1073839341, "version": 109, "versionNonce": 425519949, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555656970, "link": null, "locked": false }, { "id": "oSDGKqINOc_jAZY5sntIr", "type": "text", "x": -21176.166373774016, "y": -11260.458340739673, "width": 220.992431640625, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBP", "roundness": null, "seed": 1665154147, "version": 37, "versionNonce": 1739848707, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555764559, "link": null, "locked": false, "text": "Kafka Consumption Strategy", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Kafka Consumption Strategy", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Peu0C_9bqQGf-ja2BkVY4", "type": "text", "x": -21167.358556835094, "y": -11240.58820690622, "width": 416.1287841796875, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBQ", "roundness": null, "seed": 862375363, "version": 20, "versionNonce": 446429613, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555778145, "link": null, "locked": false, "text": "A) To ensure single controlled and reliable processing:\n", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "A) To ensure single controlled and reliable processing:\n", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "g1d27KPmV6KZmaXzh7V1O", "type": "text", "x": -21168.62231540776, "y": -11219.1429870067, "width": 214.739990234375, "height": 20.121547876058198, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBR", "roundness": null, "seed": 2046672419, "version": 33, "versionNonce": 1752925091, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555817160, "link": null, "locked": false, "text": "consumer.max-poll-records=1", "fontSize": 16.09723830084656, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "consumer.max-poll-records=1", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "oKUi5z4fD3mlkGQOC3iKr", "type": "text", "x": -21167.163425260824, "y": -11194.804823018432, "width": 189.9994185808687, "height": 20.121547876058198, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBS", "roundness": null, "seed": 2086594957, "version": 68, "versionNonce": 296369165, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555817160, "link": null, "locked": false, "text": "listener.concurrency=1", "fontSize": 16.09723830084656, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "listener.concurrency=1", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "RF9D5VMDv1vUc3yfK8rfk", "type": "text", "x": -21167.81370882745, "y": -11169.167941878171, "width": 202.9882736224893, "height": 20.121547876058198, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBT", "roundness": null, "seed": 747110851, "version": 58, "versionNonce": 1647384387, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555817160, "link": null, "locked": false, "text": "listener.ack-mode=manual", "fontSize": 16.09723830084656, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "listener.ack-mode=manual", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "4bp12j4xvZ0GRIX3m9kQS", "type": "text", "x": -20668.265249210264, "y": -11267.729078064416, "width": 124.67121722545633, "height": 13.426268347778107, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBU", "roundness": null, "seed": 181118083, "version": 308, "versionNonce": 1719743085, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555867960, "link": null, "locked": false, "text": "Pause\u2013Resume Pattern:", "fontSize": 10.741014678222484, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Pause\u2013Resume Pattern:", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "saAiKoKHL4KNZezCFxqgE", "type": "text", "x": -20672.498419498526, "y": -11252.774945373267, "width": 295.4750933133526, "height": 13.426268347778107, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBV", "roundness": null, "seed": 345053485, "version": 260, "versionNonce": 590558947, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555867961, "link": null, "locked": false, "text": "Pause the Kafka listener before starting job processing.", "fontSize": 10.741014678222484, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Pause the Kafka listener before starting job processing.", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "SKzLsxMSCMRjjElOKF0dz", "type": "text", "x": -20671.30345533812, "y": -11229.388791586507, "width": 68.40965007776097, "height": 13.426268347778107, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBW", "roundness": null, "seed": 193600547, "version": 296, "versionNonce": 1172987085, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555867961, "link": null, "locked": false, "text": "While paused:", "fontSize": 10.741014678222484, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "While paused:", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "HjpPbkEPPkJ0WDh0xgKCI", "type": "text", "x": -20668.802903068903, "y": -11212.496140329933, "width": 503.6686662779656, "height": 13.426268347778107, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBX", "roundness": null, "seed": 1590651789, "version": 299, "versionNonce": 705716867, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555867961, "link": null, "locked": false, "text": "Continue sending heartbeat polls in the background, at interval KEEP_ALIVE_POLL_TIMEOUT.", "fontSize": 10.741014678222484, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Continue sending heartbeat polls in the background, at interval KEEP_ALIVE_POLL_TIMEOUT.", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "z8iLwfi8hUY2vGQzRD8mn", "type": "text", "x": -20667.2921321751, "y": -11196.252833777724, "width": 148.6988823028636, "height": 13.426268347778107, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBY", "roundness": null, "seed": 1222156227, "version": 293, "versionNonce": 125778733, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555867961, "link": null, "locked": false, "text": "Do not fetch new messages.", "fontSize": 10.741014678222484, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Do not fetch new messages.", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "NFIDPOgH67cFxFnfJ4Q3e", "type": "text", "x": -20669.0253543464, "y": -11179.36019639278, "width": 249.56789234148778, "height": 13.426268347778107, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBZ", "roundness": null, "seed": 1179939309, "version": 290, "versionNonce": 528947747, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555867961, "link": null, "locked": false, "text": "After processing completes (success or failure):", "fontSize": 10.741014678222484, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "After processing completes (success or failure):", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "0yS1dNVVnku7yJWMo7QdI", "type": "text", "x": -20664.46932397226, "y": -11162.467531264574, "width": 105.39102335456377, "height": 13.426268347778107, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBa", "roundness": null, "seed": 1289072483, "version": 314, "versionNonce": 1403604365, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555867962, "link": null, "locked": false, "text": "Resume the listener.", "fontSize": 10.741014678222484, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Resume the listener.", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "l4T8ifuQKFS6nT-KvtUjS", "type": "text", "x": -20660.846565411004, "y": -11138.432060516701, "width": 133.21031447068776, "height": 13.426268347778107, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBb", "roundness": null, "seed": 1479126093, "version": 293, "versionNonce": 277208515, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794555867962, "link": null, "locked": false, "text": "Acknowledge the message", "fontSize": 10.741014678222484, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Acknowledge the message", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "IHgOaAyyDXH0Gb35zEFgB", "type": "text", "x": -21560.05689647807, "y": -11226.999956045, "width": 234.2084197998047, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBc", "roundness": null, "seed": 49327939, "version": 15, "versionNonce": 161616877, "isDeleted": true, "boundElements": null, "updated": 1794636714099, "created": 1794556176942, "link": null, "locked": false, "text": "batching: \nmax.poll.records (default 500)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "batching: \nmax.poll.records (default 500)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "gJyEEB-GvB5qUo0d0KyUu", "type": "text", "x": -6845.438142954621, "y": -6907.040977786514, "width": 6.4000091552734375, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBd", "roundness": null, "seed": 2000040515, "version": 3, "versionNonce": 785038723, "isDeleted": true, "boundElements": null, "updated": 1794617822473, "created": 1794617819770, "link": null, "locked": false, "text": "", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "iv0OC4Xns3Fv9g87AxnKl", "type": "arrow", "x": -11623.020149028178, "y": -4387.154463416715, "width": 4.285746256511629, "height": 226.66661580403616, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBe", "roundness": { "type": 2 }, "seed": 2096720173, "version": 242, "versionNonce": 771010701, "isDeleted": false, "boundElements": null, "updated": 1794623663931, "created": 1794623480507, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 3.2739461263026897, 118.92860048482999 ], [ 4.285746256511629, 226.66661580403616 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "3QqwrISHXc-KA9n6GGqRq", "type": "rectangle", "x": -13658.734646912293, "y": -7411.678164014045, "width": 4050.0006103515625, "height": 3028.5714721679688, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBf", "roundness": { "type": 3 }, "seed": 891926467, "version": 152, "versionNonce": 942924771, "isDeleted": false, "boundElements": [ { "id": "-H4f-p1s-k0rZQgZIVikv", "type": "arrow" } ], "updated": 1794636732030, "created": 1794623490794, "link": null, "locked": false }, { "id": "5IQ1fV3iCDyRyW4q_cyaJ", "type": "text", "x": -11576.32473988918, "y": -4270.904428910889, "width": 6.4000091552734375, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBg", "roundness": null, "seed": 823571299, "version": 5, "versionNonce": 1771327341, "isDeleted": true, "boundElements": null, "updated": 1794623594341, "created": 1794623594184, "link": null, "locked": false, "text": "", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "eOMyZA4qjlH1lRSpziLZZ", "type": "text", "x": -12050.343714880662, "y": -4342.511623538924, "width": 899.5321367869618, "height": 35.178585052490256, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBh", "roundness": null, "seed": 2070623619, "version": 274, "versionNonce": 2002459011, "isDeleted": false, "boundElements": null, "updated": 1794624444576, "created": 1794623596057, "link": null, "locked": false, "text": "SFQ + 2DFQ concept to mitigate the burst scenario among slots", "fontSize": 28.142868041992184, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "SFQ + 2DFQ concept to mitigate the burst scenario among slots", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "quv8_ld4sVwAZn9EWIWRw", "type": "text", "x": -12119.082685358626, "y": -4310.9299710338655, "width": 1121.4286804199216, "height": 112.14286804199217, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBi", "roundness": null, "seed": 2076624067, "version": 573, "versionNonce": 1362145645, "isDeleted": false, "boundElements": null, "updated": 1794624447511, "created": 1794623671024, "link": null, "locked": false, "text": "although Per tenant concurrency can be applied in among slots as well , which some what mitigates\nagainst a single tenant with agressive big job push but not when multiple tenants with single big job\npush thus other priority tenant has to wait for these long CPU sec for it's turn to arrive !!! the now\napplied scenario is for more robust spread and burst mitigation, we apply additional approach", "fontSize": 22.428573608398434, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "although Per tenant concurrency can be applied in among slots as well , which some what mitigates against a single tenant with agressive big job push but not when multiple tenants with single big job push thus other priority tenant has to wait for these long CPU sec for it's turn to arrive !!! the now applied scenario is for more robust spread and burst mitigation, we apply additional approach", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "ojYPPWDN2sXampo4eVEYb", "type": "text", "x": -11824.633027452513, "y": -4113.579231113754, "width": 671.0413208007812, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBj", "roundness": null, "seed": 571738307, "version": 106, "versionNonce": 682604131, "isDeleted": true, "boundElements": null, "updated": 1794626617700, "created": 1794626303670, "link": null, "locked": false, "text": "Group Pages Cost variance inside the group(among slots also sepregation)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Group Pages Cost variance inside the group(among slots also sepregation)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "qhDfiTJ0rruwJMDE6ZpH2", "type": "text", "x": -11815.064100945476, "y": -4083.579231113754, "width": 429.87185231849975, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBk", "roundness": null, "seed": 1067456749, "version": 171, "versionNonce": 1142396237, "isDeleted": true, "boundElements": null, "updated": 1794626617700, "created": 1794626303670, "link": null, "locked": false, "text": "sm 1\u201310 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "sm 1\u201310 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "sgKc_h-r1xrg-_gdKbDlU", "type": "text", "x": -11818.855526545323, "y": -4053.579231113754, "width": 428.40046229628723, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBl", "roundness": null, "seed": 482080867, "version": 155, "versionNonce": 1523642883, "isDeleted": true, "boundElements": null, "updated": 1794626617700, "created": 1794626303670, "link": null, "locked": false, "text": "md 11\u2013100 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "md 11\u2013100 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "72sN4XlpSoFcxWeg0Y5ZB", "type": "text", "x": -11817.728576948019, "y": -4023.579231113754, "width": 447.8767420342572, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBm", "roundness": null, "seed": 392721229, "version": 169, "versionNonce": 745628589, "isDeleted": true, "boundElements": null, "updated": 1794626617700, "created": 1794626303670, "link": null, "locked": false, "text": "lg 101\u20131000 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "lg 101\u20131000 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "I0vmXYdG19XyaHAi-WlR6", "type": "text", "x": -12023.495983619963, "y": -3896.8294116571433, "width": 1360.2166152329742, "height": 34.00541538082435, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBn", "roundness": null, "seed": 830486925, "version": 323, "versionNonce": 1671583981, "isDeleted": false, "boundElements": null, "updated": 1794626702663, "created": 1794626447737, "link": null, "locked": false, "text": "## The solution:", "fontSize": 27.20433230465948, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "## The solution:", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "s-uqcdyK02X_bAGJ9iNV1", "type": "text", "x": -12027.40575719408, "y": -4085.422233971283, "width": 646.4935913085938, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBo", "roundness": null, "seed": 11398957, "version": 93, "versionNonce": 1411248067, "isDeleted": false, "boundElements": null, "updated": 1794626621911, "created": 1794626567103, "link": null, "locked": false, "text": "Without a burst (what you want)\nslot 0 Big1 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 Big2 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\nslot 1 Big3 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 Big4 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\nslot 2 A A A A A A A A A A A A A A A A A A A A A A A A A A A A\nslot 3 Big2 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 Big4 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 Big1 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Without a burst (what you want)\nslot 0 Big1 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 Big2 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\nslot 1 Big3 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 Big4 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\nslot 2 A A A A A A A A A A A A A A A A A A A A A A A A A A A A\nslot 3 Big2 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 Big4 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 Big1 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "TkJUvvsYSogZghI3qd4DX", "type": "text", "x": -11334.205851233457, "y": -4125.717353740124, "width": 800, "height": 180, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBp", "roundness": null, "seed": 1414648589, "version": 150, "versionNonce": 1667831715, "isDeleted": false, "boundElements": null, "updated": 1794626625092, "created": 1794626574550, "link": null, "locked": false, "text": "With a burst (what SFQ gives)\nAt time 0, the four Big tenants have the lowest tags. The four slots are idle. Each slot takes one\nBig job.\n\nslot 0 Big1 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 A A A\nslot 1 Big2 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 A A A\nslot 2 Big3 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 A A A\nslot 3 Big4 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 A A A\n 0 s 60 s", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "With a burst (what SFQ gives)\nAt time 0, the four Big tenants have the lowest tags. The four slots are idle. Each slot takes one Big job.\n\nslot 0 Big1 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 A A A\nslot 1 Big2 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 A A A\nslot 2 Big3 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 A A A\nslot 3 Big4 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 A A A\n 0 s 60 s", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "yIAjjkVPR-NsqRsWNNEdQ", "type": "text", "x": -11909.095338918527, "y": -3629.856294043677, "width": 671.0413208007812, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBq", "roundness": null, "seed": 1677875331, "version": 109, "versionNonce": 461335757, "isDeleted": true, "boundElements": [], "updated": 1794636435774, "created": 1794626618928, "link": null, "locked": false, "text": "Group Pages Cost variance inside the group(among slots also sepregation)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Group Pages Cost variance inside the group(among slots also sepregation)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "Cb9rxJfed0qcegnZ6jePu", "type": "text", "x": -11899.095338918527, "y": -3619.856294043677, "width": 671.0413208007812, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBqV", "roundness": null, "seed": 250699523, "version": 111, "versionNonce": 2031552941, "isDeleted": true, "boundElements": [], "updated": 1794636435030, "created": 1794636434160, "link": null, "locked": false, "text": "Group Pages Cost variance inside the group(among slots also sepregation)", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Group Pages Cost variance inside the group(among slots also sepregation)", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "98eVWiBxs4wSGk1MOdsNs", "type": "text", "x": -11899.526412411487, "y": -3599.856294043677, "width": 429.87185231849975, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBr", "roundness": null, "seed": 1942638627, "version": 174, "versionNonce": 1880380035, "isDeleted": true, "boundElements": [], "updated": 1794636435774, "created": 1794626618928, "link": null, "locked": false, "text": "sm 1\u201310 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "sm 1\u201310 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "PvDjo5BG09NgSUbXuNOJQ", "type": "text", "x": -11889.526412411487, "y": -3589.856294043677, "width": 429.87185231849975, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBrV", "roundness": null, "seed": 672433827, "version": 176, "versionNonce": 251229091, "isDeleted": true, "boundElements": [], "updated": 1794636435030, "created": 1794636434160, "link": null, "locked": false, "text": "sm 1\u201310 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "sm 1\u201310 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "s2_tnCBQNL632k45oS6EZ", "type": "text", "x": -11903.317838011335, "y": -3569.856294043677, "width": 428.40046229628723, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBs", "roundness": null, "seed": 586306499, "version": 158, "versionNonce": 133319469, "isDeleted": true, "boundElements": [], "updated": 1794636435774, "created": 1794626618928, "link": null, "locked": false, "text": "md 11\u2013100 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "md 11\u2013100 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "WpMUp0QA_gYZ0P2AwIseb", "type": "text", "x": -11893.317838011335, "y": -3559.856294043677, "width": 428.40046229628723, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBsV", "roundness": null, "seed": 1840364099, "version": 160, "versionNonce": 239291405, "isDeleted": true, "boundElements": [], "updated": 1794636435030, "created": 1794636434161, "link": null, "locked": false, "text": "md 11\u2013100 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "md 11\u2013100 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "sAb_1RhZvZf12m91vrsJw", "type": "text", "x": -11902.190888414032, "y": -3539.856294043677, "width": 447.8767420342572, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBt", "roundness": null, "seed": 563667811, "version": 172, "versionNonce": 1603078691, "isDeleted": true, "boundElements": [], "updated": 1794636435774, "created": 1794626618928, "link": null, "locked": false, "text": "lg 101\u20131000 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "lg 101\u20131000 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "kAniFwPqThQyzC86v-TZq", "type": "text", "x": -11892.190888414032, "y": -3529.856294043677, "width": 447.8767420342572, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dotted", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBtV", "roundness": null, "seed": 1554479587, "version": 174, "versionNonce": 1668351811, "isDeleted": true, "boundElements": [], "updated": 1794636435030, "created": 1794636434161, "link": null, "locked": false, "text": "lg 101\u20131000 10\u00d7", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "lg 101\u20131000 10\u00d7", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "-oVj5hzaASra_79Ky-gba", "type": "rectangle", "x": -12038.94434957003, "y": -4158.119652967269, "width": 1534.390308480757, "height": 222.49101286332254, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBu", "roundness": { "type": 3 }, "seed": 114019043, "version": 174, "versionNonce": 484547981, "isDeleted": false, "boundElements": null, "updated": 1794626637820, "created": 1794626630676, "link": null, "locked": false }, { "id": "aAnOe4sIWp9U-oFOkL5gc", "type": "text", "x": -12025.470822031664, "y": -4152.1641159780775, "width": 214.70925049648108, "height": 32.409857894716886, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBv", "roundness": null, "seed": 1845895331, "version": 26, "versionNonce": 1691305603, "isDeleted": false, "boundElements": null, "updated": 1794626651130, "created": 1794626642338, "link": null, "locked": false, "text": "## The Problem:", "fontSize": 25.9278863157735, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "## The Problem:", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "xMNw1WNZTPQsR0-k1cNK9", "type": "rectangle", "x": -12035.526877380596, "y": -3916.2555930534427, "width": 1528.1854552771583, "height": 153.77705915177194, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBw", "roundness": { "type": 3 }, "seed": 19868045, "version": 229, "versionNonce": 363538339, "isDeleted": false, "boundElements": null, "updated": 1794636438971, "created": 1794626659838, "link": null, "locked": false }, { "id": "uhe7Gpfgv3MevvcHDAgc0", "type": "line", "x": -11360.834235824765, "y": -4159.892473578565, "width": 0, "height": 222.49101286332143, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBx", "roundness": { "type": 2 }, "seed": 646580003, "version": 55, "versionNonce": 741408579, "isDeleted": false, "boundElements": null, "updated": 1794626674767, "created": 1794626673690, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 222.49101286332143 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "polygon": false }, { "id": "l89LGjAAtKCHnvOjMNQqw", "type": "text", "x": -12019.62048750407, "y": -3857.1640167598102, "width": 800, "height": 80, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBy", "roundness": null, "seed": 1925135267, "version": 62, "versionNonce": 1957247661, "isDeleted": false, "boundElements": null, "updated": 1794636431217, "created": 1794626705091, "link": null, "locked": false, "text": "1. Per group CPU sec cap ! if more then timeout and job requeued as another slot or group(md,lg,etc\ni.e higher)\n2. Segregating the slots as well based on page ranges !!!\n3. 2DFQ implementation", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "1. Per group CPU sec cap ! if more then timeout and job requeued as another slot or group(md,lg,etc i.e higher)\n2. Segregating the slots as well based on page ranges !!!\n3. 2DFQ implementation", "autoResize": false, "lineHeight": 1.25, "labelPosition": null }, { "id": "sdsp1JTjdm3vfJt04KVM_", "type": "arrow", "x": -10578.087626453313, "y": -4348.877658013082, "width": 165.37752478589027, "height": 153.20247553226272, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bBz", "roundness": { "type": 2 }, "seed": 1822051949, "version": 53, "versionNonce": 980541283, "isDeleted": true, "boundElements": null, "updated": 1794636444361, "created": 1794636442349, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 165.37752478589027, 153.20247553226272 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "4HDrWQiZgD8NWoZAmdGnL", "type": "arrow", "x": -10347.776332256939, "y": -4065.80812048549, "width": 253.6467944288779, "height": 291.18619368222335, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bC0", "roundness": { "type": 2 }, "seed": 961374893, "version": 133, "versionNonce": 1508280525, "isDeleted": false, "boundElements": null, "updated": 1794636476469, "created": 1794636445483, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -43.62746572747528, -275.9673929521359 ], [ -253.6467944288779, -291.18619368222335 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "tkCkLQPwwNYceaIV6TVhN", "type": "arrow", "x": -10339.659892841233, "y": -4012.1734978647814, "width": 184.65463191940034, "height": 332.92268184546094, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bC1", "roundness": { "type": 2 }, "seed": 1778243395, "version": 129, "versionNonce": 1813975363, "isDeleted": false, "boundElements": [], "updated": 1794636474815, "created": 1794636447403, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 22.320815262184624, 325.4771260796639 ], [ -162.33381665721572, 332.92268184546094 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "7Q5uND4Izi8EIB3MF0WtZ", "type": "text", "x": -10312.068356873617, "y": -3757.0294724108435, "width": 6.4000091552734375, "height": 20, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bC1V", "roundness": null, "seed": 1423392525, "version": 4, "versionNonce": 1853340259, "isDeleted": true, "boundElements": null, "updated": 1794636474328, "created": 1794636474286, "link": null, "locked": false, "text": "", "fontSize": 16, "baseFontSize": null, "fontFamily": 5, "textAlign": "center", "verticalAlign": "middle", "containerId": "tkCkLQPwwNYceaIV6TVhN", "originalText": "", "autoResize": true, "lineHeight": 1.25, "labelPosition": 0.5 }, { "id": "hO74w-FMymPR6kZpJ9396", "type": "text", "x": -10488.804192013431, "y": -4057.545611626728, "width": 554.930561147352, "height": 33.18956083280412, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bC2", "roundness": null, "seed": 1461240621, "version": 88, "versionNonce": 1923773123, "isDeleted": false, "boundElements": null, "updated": 1794636481822, "created": 1794636461219, "link": null, "locked": false, "text": "Not in the scope of this project though !!!", "fontSize": 26.55164866624329, "baseFontSize": null, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Not in the scope of this project though !!!", "autoResize": true, "lineHeight": 1.25, "labelPosition": null }, { "id": "-H4f-p1s-k0rZQgZIVikv", "type": "arrow", "x": -10658.345539244108, "y": -6964.00108103832, "width": 1228.8456273199645, "height": 1045.55312672307, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bC3", "roundness": { "type": 2 }, "seed": 700811523, "version": 106, "versionNonce": 974800589, "isDeleted": false, "boundElements": null, "updated": 1794636739297, "created": 1794636732028, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1228.8456273199645, -1045.55312672307 ] ], "startBinding": { "elementId": "3CYrK5zouP0Isvp3gNbcB", "mode": "orbit", "fixedPoint": [ 0.5569879281639489, 0.55698792816395 ] }, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "1RqZNnK9HuGG00QRLLbeP", "type": "text", "x": -7625, "y": -9580, "width": 599.6, "height": 40.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ0", "roundness": null, "seed": 879521324, "version": 1, "versionNonce": 780260974, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "v5: SUPERSEDED. See panel 'Dispatch transaction (v5)'.\nattempt + 1 on each dispatch, cap unit claimed, state = 'assigned'.", "originalText": "v5: SUPERSEDED. See panel 'Dispatch transaction (v5)'.\nattempt + 1 on each dispatch, cap unit claimed, state = 'assigned'.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "Pnc2krLIX8gc0h8djdzeN", "type": "text", "x": -6110, "y": -9330, "width": 678.8000000000001, "height": 40.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ1", "roundness": null, "seed": 2066817248, "version": 1, "versionNonce": 2029536758, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "v5: SUPERSEDED. See panel 'Reaper (v5)'. One column\n(lease_expires_at); max_attempts -> Failed; cancel -> Cancelled; timeout_at.", "originalText": "v5: SUPERSEDED. See panel 'Reaper (v5)'. One column\n(lease_expires_at); max_attempts -> Failed; cancel -> Cancelled; timeout_at.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "M7W2jp2rkm1v5NPFVxijW", "type": "text", "x": -6806, "y": -9900, "width": 582.0, "height": 40.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ2", "roundness": null, "seed": 1987924597, "version": 1, "versionNonce": 464083541, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "v5: the singleton lock on scheduler_state serializes dispatch.\nReplicas WAIT; they do not pick jobs in parallel. No SKIP LOCKED.", "originalText": "v5: the singleton lock on scheduler_state serializes dispatch.\nReplicas WAIT; they do not pick jobs in parallel. No SKIP LOCKED.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "lb4tBQhkIhZBdedMAJdzK", "type": "text", "x": -8344, "y": -7680, "width": 485.20000000000005, "height": 20.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ3", "roundness": null, "seed": 208529982, "version": 1, "versionNonce": 347712950, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "v5: SUPERSEDED. See panel 'Dispatch transaction (v5)'.", "originalText": "v5: SUPERSEDED. See panel 'Dispatch transaction (v5)'.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "p4UiWwrI74-gQGGqDLtlJ", "type": "text", "x": -8281, "y": -6440, "width": 661.2, "height": 20.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ4", "roundness": null, "seed": 252417875, "version": 1, "versionNonce": 1806660056, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "v5: stream break = warning, not verdict. See panel 'Worker protocol (v5)'.", "originalText": "v5: stream break = warning, not verdict. See panel 'Worker protocol (v5)'.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "bIwLw_ezZbypIEmM-hNB_", "type": "text", "x": -12025, "y": -3740, "width": 538.0, "height": 40.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ5", "roundness": null, "seed": 1169563123, "version": 1, "versionNonce": 1551884377, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "v5: still OPEN. First step: cap in CPU seconds in flight per\ntenant, not only a job count. See panel 'Still open (v5)'.", "originalText": "v5: still OPEN. First step: cap in CPU seconds in flight per\ntenant, not only a job count. See panel 'Still open (v5)'.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "8j4dkZ_XbhM1yWfZxajiZ", "type": "text", "x": -10881, "y": -4370, "width": 423.6, "height": 40.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ6", "roundness": null, "seed": 1484262667, "version": 1, "versionNonce": 1754361097, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "v5: '0. Fault tolerant' also needs Postgres HA.\nSee panel 'Operations & security (v5)'.", "originalText": "v5: '0. Fault tolerant' also needs Postgres HA.\nSee panel 'Operations & security (v5)'.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "iMvkgILPkzaArLrcY3PKu", "type": "text", "x": -4900.0, "y": -10499, "width": 617.2, "height": 60.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ7", "roundness": null, "seed": 776284203, "version": 1, "versionNonce": 940761754, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "v5 changes (2026-11-14)", "originalText": "v5 changes (2026-11-14)", "fontSize": 48, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "N5ySL6HEeNzO4rxim9QUl", "type": "text", "x": -4900.0, "y": -10420, "width": 1099.0, "height": 50.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ8", "roundness": null, "seed": 2837325, "version": 1, "versionNonce": 1505473591, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Decisions taken: Prefer-Fit steal, tag at dispatch from tenant.finish_tag, singleton dispatch lock,\nself-kill + fencing, gRPC bidirectional stream. Red notes on the old diagram point to these panels.", "originalText": "Decisions taken: Prefer-Fit steal, tag at dispatch from tenant.finish_tag, singleton dispatch lock,\nself-kill + fencing, gRPC bidirectional stream. Red notes on the old diagram point to these panels.", "fontSize": 20, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "2ed_aIz3FuVb5Ho1zZVy4", "type": "rectangle", "x": -4900.0, "y": -10300.0, "width": 1250.0, "height": 460.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZ9", "roundness": { "type": 3 }, "seed": 1055102439, "version": 1, "versionNonce": 2105044261, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false }, { "id": "pZqEl4cCbhbjGrqlxIKmz", "type": "text", "x": -4870.0, "y": -10270.0, "width": 225.60000000000002, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZA", "roundness": null, "seed": 855374449, "version": 1, "versionNonce": 571157806, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Decisions (v5)", "originalText": "Decisions (v5)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "Bi0mDfIfmstS-bzmnKXc-", "type": "text", "x": -4870.0, "y": -10220.0, "width": 956.0000000000001, "height": 350.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZB", "roundness": null, "seed": 1078330065, "version": 1, "versionNonce": 1784523333, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "1. Steal policy: Prefer-Fit (as in Key Decision 5). Reserve{M} is optional.\n2. Tagging: tags are PER TENANT.\n start_tag = GREATEST(virtual_now, tenant.finish_tag) -- at dispatch\n tenant.finish_tag = start_tag + est_cost / weight(tier)\n All queued jobs of one tenant share the same next tag, so FIFO inside the tenant.\n3. Dispatch concurrency: singleton lock on scheduler_state. No SKIP LOCKED.\n The FOR UPDATE on one row gives mutual exclusion, so no leader lease.\n4. Zombies: self-kill + fencing.\n worker_self_kill_after (25 s) < lease_timeout (30 s)\n5. Transport: one gRPC bidirectional stream per worker.\n6. One container = one job = one execution slot. Credit is 0 or 1.\n7. attempt + 1 on EACH dispatch (fencing token).\n failures + 1 on each lost or failed run; compare failures with max_attempts.\n An Assign that is never acknowledged does not count as a failure.", "originalText": "1. Steal policy: Prefer-Fit (as in Key Decision 5). Reserve{M} is optional.\n2. Tagging: tags are PER TENANT.\n start_tag = GREATEST(virtual_now, tenant.finish_tag) -- at dispatch\n tenant.finish_tag = start_tag + est_cost / weight(tier)\n All queued jobs of one tenant share the same next tag, so FIFO inside the tenant.\n3. Dispatch concurrency: singleton lock on scheduler_state. No SKIP LOCKED.\n The FOR UPDATE on one row gives mutual exclusion, so no leader lease.\n4. Zombies: self-kill + fencing.\n worker_self_kill_after (25 s) < lease_timeout (30 s)\n5. Transport: one gRPC bidirectional stream per worker.\n6. One container = one job = one execution slot. Credit is 0 or 1.\n7. attempt + 1 on EACH dispatch (fencing token).\n failures + 1 on each lost or failed run; compare failures with max_attempts.\n An Assign that is never acknowledged does not count as a failure.", "fontSize": 20, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "Js68TYA83gs8p_6FFSEHb", "type": "rectangle", "x": -4900.0, "y": -9780.0, "width": 1250.0, "height": 785.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZC", "roundness": { "type": 3 }, "seed": 1881068557, "version": 1, "versionNonce": 643085486, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false }, { "id": "YJab-yF2Gcahb3xikG40W", "type": "text", "x": -4870.0, "y": -9750.0, "width": 395.00000000000006, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZD", "roundness": null, "seed": 1956846045, "version": 1, "versionNonce": 462141185, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Dispatch transaction (v5)", "originalText": "Dispatch transaction (v5)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "O1ssznyoPbx8c0taydILz", "type": "text", "x": -4870.0, "y": -9700.0, "width": 820.0, "height": 675.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZE", "roundness": null, "seed": 2137527558, "version": 1, "versionNonce": 1420504617, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "BEGIN;\n-- 1. singleton lock: one dispatch at a time, all replicas\nSELECT virtual_now FROM scheduler_state WHERE id = 1 FOR UPDATE;\n\n-- 2. pick: Prefer-Fit, then lowest tenant tag, then tier, then FIFO\nSELECT j.id, j.tenant_id, j.est_cost,\n GREATEST($virtual_now, t.finish_tag) AS start_tag\nFROM jobs j JOIN tenants t ON t.id = j.tenant_id\nWHERE j.state = 'queued'\n AND j.size <= $worker_size\n AND t.running < t.cap -- a cap unit is free\nORDER BY (j.size = $worker_size) DESC, -- Prefer-Fit\n start_tag, t.tier DESC, j.enqueue_seq\nLIMIT 1;\n\n-- 3. claim the cap unit and advance the tenant tag\nUPDATE tenants\nSET running = running + 1,\n finish_tag = $start_tag + $est_cost / weight(tier)\nWHERE id = $tenant AND running < cap; -- 0 rows -> ROLLBACK\n\n-- 4. hand the job out (not Running until the worker says Started)\nUPDATE jobs\nSET state = 'assigned', attempt = attempt + 1, start_tag = $start_tag,\n worker_id = $worker, ack_deadline = now() + assign_ack_timeout\nWHERE id = $job AND state = 'queued';\n\n-- 5. virtual clock never goes back\nUPDATE scheduler_state SET virtual_now = GREATEST(virtual_now, $start_tag);\nCOMMIT; -- then send Assign{job, attempt} on the worker stream", "originalText": "BEGIN;\n-- 1. singleton lock: one dispatch at a time, all replicas\nSELECT virtual_now FROM scheduler_state WHERE id = 1 FOR UPDATE;\n\n-- 2. pick: Prefer-Fit, then lowest tenant tag, then tier, then FIFO\nSELECT j.id, j.tenant_id, j.est_cost,\n GREATEST($virtual_now, t.finish_tag) AS start_tag\nFROM jobs j JOIN tenants t ON t.id = j.tenant_id\nWHERE j.state = 'queued'\n AND j.size <= $worker_size\n AND t.running < t.cap -- a cap unit is free\nORDER BY (j.size = $worker_size) DESC, -- Prefer-Fit\n start_tag, t.tier DESC, j.enqueue_seq\nLIMIT 1;\n\n-- 3. claim the cap unit and advance the tenant tag\nUPDATE tenants\nSET running = running + 1,\n finish_tag = $start_tag + $est_cost / weight(tier)\nWHERE id = $tenant AND running < cap; -- 0 rows -> ROLLBACK\n\n-- 4. hand the job out (not Running until the worker says Started)\nUPDATE jobs\nSET state = 'assigned', attempt = attempt + 1, start_tag = $start_tag,\n worker_id = $worker, ack_deadline = now() + assign_ack_timeout\nWHERE id = $job AND state = 'queued';\n\n-- 5. virtual clock never goes back\nUPDATE scheduler_state SET virtual_now = GREATEST(virtual_now, $start_tag);\nCOMMIT; -- then send Assign{job, attempt} on the worker stream", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "FwUnxzfD3n6JsSEA_RHmj", "type": "rectangle", "x": -4900.0, "y": -8935.0, "width": 1250.0, "height": 357.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZF", "roundness": { "type": 3 }, "seed": 1009398256, "version": 1, "versionNonce": 1244690595, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false }, { "id": "NNqK8R5Dm9Z7FpH1tBe__", "type": "text", "x": -4870.0, "y": -8905.0, "width": 502.80000000000007, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZG", "roundness": null, "seed": 1517853884, "version": 1, "versionNonce": 1607169039, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Cost correction on Finished (v5)", "originalText": "Cost correction on Finished (v5)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "7HbJbSX8UWaHUlwnMG2hA", "type": "text", "x": -4870.0, "y": -8855.0, "width": 841.6, "height": 247.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZH", "roundness": null, "seed": 154387790, "version": 1, "versionNonce": 391607689, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "UPDATE jobs SET state = 'completed', cpu_seconds = $cpu\nWHERE id = $job AND attempt = $attempt AND worker_id = $worker\n AND state = 'running'; -- fencing\n-- only if 1 row changed, in the same transaction:\nUPDATE tenants\nSET running = running - 1, -- release cap unit\n finish_tag = finish_tag + ($cpu - $est_cost) / weight(tier)\nWHERE id = $tenant;\n\n-- stale attempt (0 rows): do not change state,\n-- but CHARGE its cpu to the tenant (billing only), so zombie CPU is not free", "originalText": "UPDATE jobs SET state = 'completed', cpu_seconds = $cpu\nWHERE id = $job AND attempt = $attempt AND worker_id = $worker\n AND state = 'running'; -- fencing\n-- only if 1 row changed, in the same transaction:\nUPDATE tenants\nSET running = running - 1, -- release cap unit\n finish_tag = finish_tag + ($cpu - $est_cost) / weight(tier)\nWHERE id = $tenant;\n\n-- stale attempt (0 rows): do not change state,\n-- but CHARGE its cpu to the tenant (billing only), so zombie CPU is not free", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "U4z4ile72wyJFfbwUK4P3", "type": "rectangle", "x": -3590.0, "y": -10300.0, "width": 1250.0, "height": 627.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZI", "roundness": { "type": 3 }, "seed": 989871726, "version": 1, "versionNonce": 335880733, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false }, { "id": "ginmHjbtk4RNo8C44NPP5", "type": "text", "x": -3560.0, "y": -10270.0, "width": 225.60000000000002, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZJ", "roundness": null, "seed": 2104659149, "version": 1, "versionNonce": 2065845216, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Lifecycle (v5)", "originalText": "Lifecycle (v5)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "i0sycsfxxCOJK9lnyJWEZ", "type": "text", "x": -3560.0, "y": -10220.0, "width": 1036.0, "height": 517.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZK", "roundness": null, "seed": 693798164, "version": 1, "versionNonce": 1232883085, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Events: Dispatch{attempt, ack_deadline}, Started, AckTimeout, Complete, Fail,\n Cancel{reason}, LeaseExpired, TimedOut -- all carry attempt\n Times are database timestamps (now()), never a process Instant.\n\nQueued --dispatch--> Assigned{attempt, ack_deadline}\nAssigned --Started{attempt}--> Running{attempt, lease_expires_at, timeout_at, cancel_requested}\nAssigned --ack_deadline passed--> Queued (failures unchanged)\n\nRunning --Finished{Completed}--> Completed\nRunning --Finished{Failed} or LeaseExpired--> Queued if failures + 1 < max_attempts\n --> Failed otherwise\nRunning --timeout_at passed--> send Cancel{Timeout} --> TimedOut\nRunning{cancel_requested} --Finished{Cancelled} or LeaseExpired--> Cancelled\nQueued --Cancel--> Cancelled\n\nRules:\n- Each transition out of Assigned / Running releases the cap unit\n (tenants.running - 1) in the SAME transaction.\n- A requeued job keeps its enqueue_seq, so it is first in its tenant FIFO.\n The tenant already paid est_cost at dispatch; a lost run is not refunded.\n- Each worker write: WHERE id = $job AND attempt = $attempt\n AND worker_id = $worker AND state = . Equal, not \"lower than\".\n- Side effects run only when the row changed (Transition::Changed).", "originalText": "Events: Dispatch{attempt, ack_deadline}, Started, AckTimeout, Complete, Fail,\n Cancel{reason}, LeaseExpired, TimedOut -- all carry attempt\n Times are database timestamps (now()), never a process Instant.\n\nQueued --dispatch--> Assigned{attempt, ack_deadline}\nAssigned --Started{attempt}--> Running{attempt, lease_expires_at, timeout_at, cancel_requested}\nAssigned --ack_deadline passed--> Queued (failures unchanged)\n\nRunning --Finished{Completed}--> Completed\nRunning --Finished{Failed} or LeaseExpired--> Queued if failures + 1 < max_attempts\n --> Failed otherwise\nRunning --timeout_at passed--> send Cancel{Timeout} --> TimedOut\nRunning{cancel_requested} --Finished{Cancelled} or LeaseExpired--> Cancelled\nQueued --Cancel--> Cancelled\n\nRules:\n- Each transition out of Assigned / Running releases the cap unit\n (tenants.running - 1) in the SAME transaction.\n- A requeued job keeps its enqueue_seq, so it is first in its tenant FIFO.\n The tenant already paid est_cost at dispatch; a lost run is not refunded.\n- Each worker write: WHERE id = $job AND attempt = $attempt\n AND worker_id = $worker AND state = . Equal, not \"lower than\".\n- Side effects run only when the row changed (Transition::Changed).", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "pOTAeDBK9R2uj_smG1dvS", "type": "rectangle", "x": -3590.0, "y": -9612.5, "width": 1250.0, "height": 492.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZL", "roundness": { "type": 3 }, "seed": 496871123, "version": 1, "versionNonce": 595782949, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false }, { "id": "pkdNxka8xKUMQ5tY0x7vq", "type": "text", "x": -3560.0, "y": -9582.5, "width": 733.8000000000001, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZM", "roundness": null, "seed": 729102612, "version": 1, "versionNonce": 193051585, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Worker protocol: gRPC bidirectional stream (v5)", "originalText": "Worker protocol: gRPC bidirectional stream (v5)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "4iC9U_By6V6-o76Wqvd64", "type": "text", "x": -3560.0, "y": -9532.5, "width": 938.8, "height": 382.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZN", "roundness": null, "seed": 835359116, "version": 1, "versionNonce": 770458910, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Worker -> Dispatcher Dispatcher -> Worker\nHello{worker_id, class, running[]} Welcome{heartbeat_interval, self_kill_after,\nReady{} (1 credit) pending_cancels[]}\nStarted{job, attempt} Assign{job, attempt, timeout, payload}\nHeartbeat{job, attempt, cpu_ms} Cancel{job, attempt, reason, grace}\nFinished{job, attempt, outcome, cpu_ms} Reject{job, attempt} -> kill container\n\n- No prefetch: Assign only when the worker sent Ready. SFQ picks at the last moment.\n- Heartbeat while idle = slot heartbeat. Heartbeat with a job = lease renewal.\n- Stream break: mark leases SUSPECT, wait lease_timeout. On reconnect, Hello.running[]\n resumes the leases; pending cancels come back in Welcome.\n- Cancel is durable first (cancel_requested in DB), then sent; it is sent again on\n reconnect and on each heartbeat until Finished. Idempotent on the worker:\n Stopping -> do NOT reset the kill timer; Exited -> send Finished again.\n- Kill with the container ID (docker stop), never a raw PID.\n- No contact for worker_self_kill_after (25 s) -> worker stops its container.\n- Keepalive: HTTP/2 PING 10 s, timeout 5 s.", "originalText": "Worker -> Dispatcher Dispatcher -> Worker\nHello{worker_id, class, running[]} Welcome{heartbeat_interval, self_kill_after,\nReady{} (1 credit) pending_cancels[]}\nStarted{job, attempt} Assign{job, attempt, timeout, payload}\nHeartbeat{job, attempt, cpu_ms} Cancel{job, attempt, reason, grace}\nFinished{job, attempt, outcome, cpu_ms} Reject{job, attempt} -> kill container\n\n- No prefetch: Assign only when the worker sent Ready. SFQ picks at the last moment.\n- Heartbeat while idle = slot heartbeat. Heartbeat with a job = lease renewal.\n- Stream break: mark leases SUSPECT, wait lease_timeout. On reconnect, Hello.running[]\n resumes the leases; pending cancels come back in Welcome.\n- Cancel is durable first (cancel_requested in DB), then sent; it is sent again on\n reconnect and on each heartbeat until Finished. Idempotent on the worker:\n Stopping -> do NOT reset the kill timer; Exited -> send Finished again.\n- Kill with the container ID (docker stop), never a raw PID.\n- No contact for worker_self_kill_after (25 s) -> worker stops its container.\n- Keepalive: HTTP/2 PING 10 s, timeout 5 s.", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "-fa4833F6GjtX2TsGkPgj", "type": "rectangle", "x": -3590.0, "y": -9060.0, "width": 1250.0, "height": 515.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZO", "roundness": { "type": 3 }, "seed": 53454792, "version": 1, "versionNonce": 701347340, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false }, { "id": "dzBXpeGGAyVAsVZ7-V6Pa", "type": "text", "x": -3560.0, "y": -9030.0, "width": 887.8000000000001, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZP", "roundness": null, "seed": 1605298452, "version": 1, "versionNonce": 423187297, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Reaper (v5): every few seconds, every replica, idempotent", "originalText": "Reaper (v5): every few seconds, every replica, idempotent", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "9rem0aG3uI26sYDmGf8dL", "type": "text", "x": -3560.0, "y": -8980.0, "width": 917.1999999999999, "height": 405.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZQ", "roundness": null, "seed": 589828455, "version": 1, "versionNonce": 1127824120, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "-- 1. Assign never acknowledged -> back to queue, not a failure\nUPDATE jobs SET state = 'queued', worker_id = NULL\nWHERE state = 'assigned' AND ack_deadline < now() RETURNING tenant_id;\n-- 2. lease expired and cancel requested -> Cancelled\nUPDATE jobs SET state = 'cancelled'\nWHERE state = 'running' AND lease_expires_at < now()\n AND cancel_requested RETURNING tenant_id;\n-- 3. lease expired -> retry or fail\nUPDATE jobs SET failures = failures + 1, worker_id = NULL,\n state = CASE WHEN failures + 1 < max_attempts THEN 'queued' ELSE 'failed' END\nWHERE state = 'running' AND lease_expires_at < now()\n AND NOT cancel_requested RETURNING tenant_id;\n-- 4. job ran too long -> ask the worker to stop (final event TimedOut)\nUPDATE jobs SET cancel_requested = true, cancel_reason = 'timeout'\nWHERE state = 'running' AND timeout_at < now() AND NOT cancel_requested;\n-- for 1-3, same transaction: tenants.running - 1 for each RETURNING row\n\nlease_expires_at: set at Started, moved forward by each heartbeat (one column only).", "originalText": "-- 1. Assign never acknowledged -> back to queue, not a failure\nUPDATE jobs SET state = 'queued', worker_id = NULL\nWHERE state = 'assigned' AND ack_deadline < now() RETURNING tenant_id;\n-- 2. lease expired and cancel requested -> Cancelled\nUPDATE jobs SET state = 'cancelled'\nWHERE state = 'running' AND lease_expires_at < now()\n AND cancel_requested RETURNING tenant_id;\n-- 3. lease expired -> retry or fail\nUPDATE jobs SET failures = failures + 1, worker_id = NULL,\n state = CASE WHEN failures + 1 < max_attempts THEN 'queued' ELSE 'failed' END\nWHERE state = 'running' AND lease_expires_at < now()\n AND NOT cancel_requested RETURNING tenant_id;\n-- 4. job ran too long -> ask the worker to stop (final event TimedOut)\nUPDATE jobs SET cancel_requested = true, cancel_reason = 'timeout'\nWHERE state = 'running' AND timeout_at < now() AND NOT cancel_requested;\n-- for 1-3, same transaction: tenants.running - 1 for each RETURNING row\n\nlease_expires_at: set at Started, moved forward by each heartbeat (one column only).", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "piIIU_m4CLoCaPfzyBPm1", "type": "rectangle", "x": -3590.0, "y": -8485.0, "width": 1250.0, "height": 535.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZR", "roundness": { "type": 3 }, "seed": 444693733, "version": 1, "versionNonce": 793450380, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false }, { "id": "KCYLIvtsFgevTxtLu6Ky_", "type": "text", "x": -3560.0, "y": -8455.0, "width": 410.40000000000003, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZS", "roundness": null, "seed": 6935555, "version": 1, "versionNonce": 444799922, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Operations & security (v5)", "originalText": "Operations & security (v5)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "Gr6Q04UzjoxRPZPcjUkjV", "type": "text", "x": -3560.0, "y": -8405.0, "width": 956.0000000000001, "height": 425.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZT", "roundness": null, "seed": 266569948, "version": 1, "versionNonce": 2123330956, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "- Postgres is a single point of failure: synchronous replica + failover.\n Leases live in the DB, so they survive; lease_timeout > failover time.\n- now() is a wall clock, not monotonic. It is one shared clock, so no skew between\n replicas. Use clock_timestamp() inside long transactions.\n- Timeouts: statement_timeout, lock_timeout, idle_in_transaction_session_timeout,\n so a paused replica cannot hold the scheduler_state lock.\n- Workers: mTLS. Each lease is bound to worker_id; worker_id is in each fencing WHERE.\n- Clients: tenant_id in each WHERE for submit / cancel / status (authorization).\n- CPU billing: read cgroup values on the host side if possible; a worker that\n reports its own cpu can bill any tenant any amount.\n- Admission control: max_queued_per_tenant; reject with an error when full.\n- Idempotency: UNIQUE (tenant_id, idempotency_key). Same key + different payload\n -> reject.\n- Status updates to the SaaS app: transactional outbox table, written in the\n same transaction as the state change; a relay sends them.\n- Indexes: ON jobs (state, size, enqueue_seq) WHERE state = 'queued';\n retention policy for terminal jobs.", "originalText": "- Postgres is a single point of failure: synchronous replica + failover.\n Leases live in the DB, so they survive; lease_timeout > failover time.\n- now() is a wall clock, not monotonic. It is one shared clock, so no skew between\n replicas. Use clock_timestamp() inside long transactions.\n- Timeouts: statement_timeout, lock_timeout, idle_in_transaction_session_timeout,\n so a paused replica cannot hold the scheduler_state lock.\n- Workers: mTLS. Each lease is bound to worker_id; worker_id is in each fencing WHERE.\n- Clients: tenant_id in each WHERE for submit / cancel / status (authorization).\n- CPU billing: read cgroup values on the host side if possible; a worker that\n reports its own cpu can bill any tenant any amount.\n- Admission control: max_queued_per_tenant; reject with an error when full.\n- Idempotency: UNIQUE (tenant_id, idempotency_key). Same key + different payload\n -> reject.\n- Status updates to the SaaS app: transactional outbox table, written in the\n same transaction as the state change; a relay sends them.\n- Indexes: ON jobs (state, size, enqueue_seq) WHERE state = 'queued';\n retention policy for terminal jobs.", "fontSize": 20, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "fDlVCRbdY9L9qNS8qrTsz", "type": "rectangle", "x": -4900.0, "y": -8517.5, "width": 1250.0, "height": 335.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZU", "roundness": { "type": 3 }, "seed": 1049457641, "version": 1, "versionNonce": 1298469150, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false }, { "id": "n2e3dnypv7Irk2Qaa93Bj", "type": "text", "x": -4870.0, "y": -8487.5, "width": 241.00000000000003, "height": 35.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZV", "roundness": null, "seed": 1774272231, "version": 1, "versionNonce": 1317663771, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "Still open (v5)", "originalText": "Still open (v5)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "1ZU94BciQzBFUhso0nmMQ", "type": "text", "x": -4870.0, "y": -8437.5, "width": 901.0000000000001, "height": 225.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "bZW", "roundness": null, "seed": 1380440956, "version": 1, "versionNonce": 1660710389, "isDeleted": false, "boundElements": [], "updated": 1794661218463, "created": 1794661218463, "link": null, "locked": false, "text": "1. Penalization / decay (history-aware fairness): no mechanism yet.\n2. Optional preemption: not defined. Prefer-Fit \"lg jobs take precedence\" means\n first pick of the NEXT free lg container, not preemption.\n3. Tier change while jobs are queued: proposal: the new weight applies from the\n next tag.\n4. Burst of big jobs (2DFQ section): first step is a cap in CPU seconds in flight\n per tenant (sum of est_cost of running jobs), not only a job count.\n5. \"Segregating slots based on page ranges\": needs a definition.\n6. Cost estimate source: est_cost at submit, or an EMA per job_type.", "originalText": "1. Penalization / decay (history-aware fairness): no mechanism yet.\n2. Optional preemption: not defined. Prefer-Fit \"lg jobs take precedence\" means\n first pick of the NEXT free lg container, not preemption.\n3. Tier change while jobs are queued: proposal: the new weight applies from the\n next tag.\n4. Burst of big jobs (2DFQ section): first step is a cap in CPU seconds in flight\n per tenant (sum of est_cost of running jobs), not only a job count.\n5. \"Segregating slots based on page ranges\": needs a definition.\n6. Cost estimate source: est_cost at submit, or an EMA per job_type.", "fontSize": 20, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "k-Heas8VOcI-z0myHl2Ql", "type": "text", "x": -2250.0, "y": -10499.0, "width": 607.2, "height": 60.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0001", "roundness": null, "seed": 777690977, "version": 1, "versionNonce": 1718594054, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "v6 changes (2026-11-15)", "originalText": "v6 changes (2026-11-15)", "fontSize": 48, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "0G4mzLmfzU-ydUF3MTplz", "type": "text", "x": -2250.0, "y": -10420.0, "width": 1023.0000000000001, "height": 75.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0002", "roundness": null, "seed": 250878467, "version": 1, "versionNonce": 1306117583, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Decision taken: history-aware fairness through usage decay (Slurm style).\nEach tenant has a usage memory that decays with a half-life and lowers its weight.\nRed notes on the old diagram point to these panels. Panels (v5) stay valid unless noted here.", "originalText": "Decision taken: history-aware fairness through usage decay (Slurm style).\nEach tenant has a usage memory that decays with a half-life and lowers its weight.\nRed notes on the old diagram point to these panels. Panels (v5) stay valid unless noted here.", "fontSize": 20, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "GUyG5QG01-I89-sW_N7PU", "type": "rectangle", "x": -2250.0, "y": -10269.0, "width": 1250, "height": 410.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0005", "roundness": { "type": 3 }, "seed": 1415713904, "version": 1, "versionNonce": 350106570, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false }, { "id": "INzW9CrAbyvcQNVWXLq-g", "type": "text", "x": -2220.0, "y": -10239.0, "width": 246.40000000000003, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0003", "roundness": null, "seed": 398033817, "version": 1, "versionNonce": 908146465, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Decay model (v6)", "originalText": "Decay model (v6)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "Ym4Fl55Wj28M0kyI5-wcc", "type": "text", "x": -2220.0, "y": -10189.0, "width": 891.0000000000001, "height": 300.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0004", "roundness": null, "seed": 1155222272, "version": 1, "versionNonce": 265204172, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Each tenant keeps one memory: usage (CPU ms) and usage_updated_at.\nNothing runs on a timer. The scheduler decays the value when it reads the tenant.\n\n elapsed = now - usage_updated_at\n usage = usage * 2^(-elapsed / half_life) -- forget, smoothly\n usage = usage + est_cost -- charge the new job\n usage_updated_at = now\n\nhalf_life: after one half-life, half of the usage is left. It is NOT a reset.\n day 0: 20000 day 7: 10000 day 14: 5000 day 21: 2500 day 28: 1250\nTwo steps (7 d then 14 d) give the same result as one step (21 d).\nThe lazy update is exact: no cron job, no cliff, no clock to play with.", "originalText": "Each tenant keeps one memory: usage (CPU ms) and usage_updated_at.\nNothing runs on a timer. The scheduler decays the value when it reads the tenant.\n\n elapsed = now - usage_updated_at\n usage = usage * 2^(-elapsed / half_life) -- forget, smoothly\n usage = usage + est_cost -- charge the new job\n usage_updated_at = now\n\nhalf_life: after one half-life, half of the usage is left. It is NOT a reset.\n day 0: 20000 day 7: 10000 day 14: 5000 day 21: 2500 day 28: 1250\nTwo steps (7 d then 14 d) give the same result as one step (21 d).\nThe lazy update is exact: no cron job, no cliff, no clock to play with.", "fontSize": 20, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "H5_cry4945J9iLUKUekJJ", "type": "rectangle", "x": -2250.0, "y": -9799.0, "width": 1250, "height": 627.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0008", "roundness": { "type": 3 }, "seed": 588657131, "version": 1, "versionNonce": 2074942856, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false }, { "id": "WIOY-uLa5hxdoVU_hytIb", "type": "text", "x": -2220.0, "y": -9769.0, "width": 292.6, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0006", "roundness": null, "seed": 906477776, "version": 1, "versionNonce": 1731332818, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Updated weight (v6)", "originalText": "Updated weight (v6)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "-j8DmVUsFOsemnf97idqk", "type": "text", "x": -2220.0, "y": -9719.0, "width": 734.4, "height": 517.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0007", "roundness": null, "seed": 345870664, "version": 1, "versionNonce": 1327406298, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "v5: weight = tier_weight(tier) * multiplier\n\nv6: factor = 2^(-usage / norm)\n weight = tier_weight(tier) * multiplier * factor\n\nnorm = the usage that halves the weight. Gold (8), norm 10000:\n usage 0 -> factor 1.00 -> weight 8\n usage 10000 -> factor 0.50 -> weight 4 (= an idle Silver)\n usage 20000 -> factor 0.25 -> weight 2 (= an idle Bronze)\n\nThe weight enters SFQ in one place only. The form is unchanged:\n start_tag = GREATEST(virtual_now, tenant.finish_tag)\n finish_tag = start_tag + est_cost / weight\nA small weight gives large tag steps, so other tenants run first.\nTags still only increase, so virtual_now stays monotonic.\n\nInteger form (S = 2^16, half_power(n, d) ~= S * 2^(-n/d)):\n decay = half_power(elapsed_ms, half_life_ms)\n usage = usage * decay / S\n factor = half_power(usage, norm_ms)\n weight = max(1, tier_weight * multiplier * factor / S)\nhalf_power = shift by the whole part, lookup table for the fraction.\nNo floats, so the replay tests stay exact on every machine.", "originalText": "v5: weight = tier_weight(tier) * multiplier\n\nv6: factor = 2^(-usage / norm)\n weight = tier_weight(tier) * multiplier * factor\n\nnorm = the usage that halves the weight. Gold (8), norm 10000:\n usage 0 -> factor 1.00 -> weight 8\n usage 10000 -> factor 0.50 -> weight 4 (= an idle Silver)\n usage 20000 -> factor 0.25 -> weight 2 (= an idle Bronze)\n\nThe weight enters SFQ in one place only. The form is unchanged:\n start_tag = GREATEST(virtual_now, tenant.finish_tag)\n finish_tag = start_tag + est_cost / weight\nA small weight gives large tag steps, so other tenants run first.\nTags still only increase, so virtual_now stays monotonic.\n\nInteger form (S = 2^16, half_power(n, d) ~= S * 2^(-n/d)):\n decay = half_power(elapsed_ms, half_life_ms)\n usage = usage * decay / S\n factor = half_power(usage, norm_ms)\n weight = max(1, tier_weight * multiplier * factor / S)\nhalf_power = shift by the whole part, lookup table for the fraction.\nNo floats, so the replay tests stay exact on every machine.", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "XyPW4bYrDVIxtvrXjhbf8", "type": "rectangle", "x": -2250.0, "y": -9111.5, "width": 1250, "height": 425.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0011", "roundness": { "type": 3 }, "seed": 1629866227, "version": 1, "versionNonce": 127189228, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false }, { "id": "834VekFEewRVegA3RjuoZ", "type": "text", "x": -2220.0, "y": -9081.5, "width": 323.40000000000003, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0009", "roundness": null, "seed": 1174943148, "version": 1, "versionNonce": 291397743, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Recovery example (v6)", "originalText": "Recovery example (v6)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "7_jqjSdxtxKtXCCq1uOEa", "type": "text", "x": -2220.0, "y": -9031.5, "width": 766.8, "height": 315.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0010", "roundness": null, "seed": 1688820160, "version": 1, "versionNonce": 1728906758, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Gold tenant, tier weight 8, half_life 7 d, norm 10000 s.\nBurst of 20000 CPU s on day 0, then idle.\n\n day usage factor weight\n 0 20000 0.25 2.0\n 7 10000 0.50 4.0\n 14 5000 0.71 5.7\n 21 2500 0.84 6.7\n 28 1250 0.92 7.3\n 35 625 0.96 7.7 -> rounds to 8: full weight again\n\nThe weight can not go above the tier weight: factor <= 1, usage >= 0.\nA tenant that keeps running keeps adding usage, so it does not recover.\nThat is the intent: the penalty follows the load, not the calendar.", "originalText": "Gold tenant, tier weight 8, half_life 7 d, norm 10000 s.\nBurst of 20000 CPU s on day 0, then idle.\n\n day usage factor weight\n 0 20000 0.25 2.0\n 7 10000 0.50 4.0\n 14 5000 0.71 5.7\n 21 2500 0.84 6.7\n 28 1250 0.92 7.3\n 35 625 0.96 7.7 -> rounds to 8: full weight again\n\nThe weight can not go above the tier weight: factor <= 1, usage >= 0.\nA tenant that keeps running keeps adding usage, so it does not recover.\nThat is the intent: the penalty follows the load, not the calendar.", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "dbRi-a7HlIDqq1xFX29IG", "type": "rectangle", "x": -2250.0, "y": -8626.5, "width": 1250, "height": 310.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0014", "roundness": { "type": 3 }, "seed": 293718147, "version": 1, "versionNonce": 642219529, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false }, { "id": "omVehC0CLl8jhv4powgd2", "type": "text", "x": -2220.0, "y": -8596.5, "width": 231.00000000000003, "height": 35.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0012", "roundness": null, "seed": 1397935985, "version": 1, "versionNonce": 1521110737, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Still open (v6)", "originalText": "Still open (v6)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "KG4DzBJbGQleUUoCdUR3g", "type": "text", "x": -2220.0, "y": -8546.5, "width": 869.0000000000001, "height": 200.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0013", "roundness": null, "seed": 738751746, "version": 1, "versionNonce": 299357824, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "1. Penalization / decay: RESOLVED in v6. See the panels above.\n2. Optional preemption: not defined.\n3. Tier change while jobs are queued: the new weight applies from the next tag.\n v6 gives this for free: the weight is calculated again at each dispatch.\n4. Burst of big jobs (2DFQ): cap in CPU seconds in flight per tenant.\n5. \"Segregating slots based on page ranges\": needs a definition.\n6. Cost estimate source: est_cost at submit, or an EMA per job_type.\n7. Per-tier norm: one norm bites heavy tiers more than Free. Decide with data.", "originalText": "1. Penalization / decay: RESOLVED in v6. See the panels above.\n2. Optional preemption: not defined.\n3. Tier change while jobs are queued: the new weight applies from the next tag.\n v6 gives this for free: the weight is calculated again at each dispatch.\n4. Burst of big jobs (2DFQ): cap in CPU seconds in flight per tenant.\n5. \"Segregating slots based on page ranges\": needs a definition.\n6. Cost estimate source: est_cost at submit, or an EMA per job_type.\n7. Per-tier norm: one norm bites heavy tiers more than Free. Decide with data.", "fontSize": 20, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "XIiHqfotollpBSRuFwxCq", "type": "rectangle", "x": -940.0, "y": -10269.0, "width": 1250, "height": 852.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0017", "roundness": { "type": 3 }, "seed": 1845767333, "version": 1, "versionNonce": 1028805850, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false }, { "id": "ey5nwnauptY8-8BXt6bR1", "type": "text", "x": -910.0, "y": -10239.0, "width": 385.00000000000006, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0015", "roundness": null, "seed": 540784943, "version": 1, "versionNonce": 685824041, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Dispatch transaction (v6)", "originalText": "Dispatch transaction (v6)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "rKVUCaNwk0DlyW0qsg-UQ", "type": "text", "x": -910.0, "y": -10189.0, "width": 831.6, "height": 742.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0016", "roundness": null, "seed": 1807541045, "version": 1, "versionNonce": 1529855948, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "-- Same as v5. Only the tenant weight is now read through the decay.\n-- decayed_usage(t, now()) and weight_of(t, now()) are SQL functions\n-- that use the same integer half_power table as barista-core.\nBEGIN;\nSELECT virtual_now FROM scheduler_state WHERE id = 1 FOR UPDATE;\n\n-- 2. pick (ranking unchanged): Prefer-Fit, start_tag, tier, enqueue_seq\nSELECT j.id, j.tenant_id, j.est_cost,\n GREATEST($virtual_now, t.finish_tag) AS start_tag,\n weight_of(t, now()) AS weight -- NEW\nFROM jobs j JOIN tenants t ON t.id = j.tenant_id\nWHERE j.state = 'queued' AND j.size <= $worker_size\n AND t.running < t.cap\nORDER BY (j.size = $worker_size) DESC, start_tag, t.tier DESC, j.enqueue_seq\nLIMIT 1;\n\n-- 3. claim the cap unit, decay + charge the usage, advance the tag\nUPDATE tenants t\nSET running = running + 1,\n usage = decayed_usage(t, now()) + $est_cost, -- NEW\n usage_updated_at = now(), -- NEW\n finish_tag = $start_tag + $est_cost / $weight\nWHERE id = $tenant AND running < cap; -- 0 rows -> ROLLBACK\n\n-- 4. hand the job out; keep the weight used, for the correction later\nUPDATE jobs SET state = 'assigned', attempt = attempt + 1,\n start_tag = $start_tag, weight_at_dispatch = $weight, -- NEW\n worker_id = $worker, ack_deadline = now() + assign_ack_timeout\nWHERE id = $job AND state = 'queued';\n\n-- 5. unchanged\nUPDATE scheduler_state SET virtual_now = GREATEST(virtual_now, $start_tag);\nCOMMIT;", "originalText": "-- Same as v5. Only the tenant weight is now read through the decay.\n-- decayed_usage(t, now()) and weight_of(t, now()) are SQL functions\n-- that use the same integer half_power table as barista-core.\nBEGIN;\nSELECT virtual_now FROM scheduler_state WHERE id = 1 FOR UPDATE;\n\n-- 2. pick (ranking unchanged): Prefer-Fit, start_tag, tier, enqueue_seq\nSELECT j.id, j.tenant_id, j.est_cost,\n GREATEST($virtual_now, t.finish_tag) AS start_tag,\n weight_of(t, now()) AS weight -- NEW\nFROM jobs j JOIN tenants t ON t.id = j.tenant_id\nWHERE j.state = 'queued' AND j.size <= $worker_size\n AND t.running < t.cap\nORDER BY (j.size = $worker_size) DESC, start_tag, t.tier DESC, j.enqueue_seq\nLIMIT 1;\n\n-- 3. claim the cap unit, decay + charge the usage, advance the tag\nUPDATE tenants t\nSET running = running + 1,\n usage = decayed_usage(t, now()) + $est_cost, -- NEW\n usage_updated_at = now(), -- NEW\n finish_tag = $start_tag + $est_cost / $weight\nWHERE id = $tenant AND running < cap; -- 0 rows -> ROLLBACK\n\n-- 4. hand the job out; keep the weight used, for the correction later\nUPDATE jobs SET state = 'assigned', attempt = attempt + 1,\n start_tag = $start_tag, weight_at_dispatch = $weight, -- NEW\n worker_id = $worker, ack_deadline = now() + assign_ack_timeout\nWHERE id = $job AND state = 'queued';\n\n-- 5. unchanged\nUPDATE scheduler_state SET virtual_now = GREATEST(virtual_now, $start_tag);\nCOMMIT;", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "v4QGSLI5fFa_GvePbAMyt", "type": "rectangle", "x": -940.0, "y": -9356.5, "width": 1250, "height": 447.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0020", "roundness": { "type": 3 }, "seed": 597137594, "version": 1, "versionNonce": 125011107, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false }, { "id": "6or6zVsQHiJahd96OhbJm", "type": "text", "x": -910.0, "y": -9326.5, "width": 492.80000000000007, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0018", "roundness": null, "seed": 824786606, "version": 1, "versionNonce": 2004949450, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Cost correction on Finished (v6)", "originalText": "Cost correction on Finished (v6)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "RnVssV8Zctn5Vbq9Hj9pZ", "type": "text", "x": -910.0, "y": -9276.5, "width": 864.0, "height": 337.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0019", "roundness": null, "seed": 1513983921, "version": 1, "versionNonce": 426400390, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "UPDATE jobs SET state = 'completed', cpu_seconds = $cpu\nWHERE id = $job AND attempt = $attempt AND worker_id = $worker\n AND state = 'running'; -- fencing, unchanged\n-- only if 1 row changed, in the same transaction:\nUPDATE tenants t\nSET running = running - 1,\n finish_tag = finish_tag + ($cpu - $est_cost) / $weight_at_dispatch,\n usage = decayed_usage(t, now()) + ($cpu - $est_cost), -- NEW\n usage_updated_at = now() -- NEW\nWHERE id = $tenant;\n\n-- Use the weight stored at dispatch, not a fresh one. The tag step was\n-- taken with that weight, so the correction must use the same divisor.\n-- Stale attempt (0 rows): no state change, but charge the usage too.\n-- Zombie CPU is not free.", "originalText": "UPDATE jobs SET state = 'completed', cpu_seconds = $cpu\nWHERE id = $job AND attempt = $attempt AND worker_id = $worker\n AND state = 'running'; -- fencing, unchanged\n-- only if 1 row changed, in the same transaction:\nUPDATE tenants t\nSET running = running - 1,\n finish_tag = finish_tag + ($cpu - $est_cost) / $weight_at_dispatch,\n usage = decayed_usage(t, now()) + ($cpu - $est_cost), -- NEW\n usage_updated_at = now() -- NEW\nWHERE id = $tenant;\n\n-- Use the weight stored at dispatch, not a fresh one. The tag step was\n-- taken with that weight, so the correction must use the same divisor.\n-- Stale attempt (0 rows): no state change, but charge the usage too.\n-- Zombie CPU is not free.", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "lwHTaiINSXZjoK_5GNEbg", "type": "rectangle", "x": -940.0, "y": -8849.0, "width": 1250, "height": 447.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0023", "roundness": { "type": 3 }, "seed": 1664584618, "version": 1, "versionNonce": 1106741744, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false }, { "id": "IMVBvalTxSx9C8XTpl3gA", "type": "text", "x": -910.0, "y": -8819.0, "width": 200.20000000000002, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0021", "roundness": null, "seed": 1307729652, "version": 1, "versionNonce": 942939739, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Settings (v6)", "originalText": "Settings (v6)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "ICh9GIUM2Vm1uJhOhkoh4", "type": "text", "x": -910.0, "y": -8769.0, "width": 939.5999999999999, "height": 337.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0022", "roundness": null, "seed": 924611747, "version": 1, "versionNonce": 1145928327, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Setting Scope Set by Changes at runtime\nhalf_life whole scheduler config no (Slurm PriorityDecayHalfLife, 7 d)\nnorm whole scheduler config no\nmultiplier one tenant operator only when the operator changes it\nusage one tenant scheduler at each dispatch and correction\nusage_updated_at one tenant scheduler at each dispatch and correction\n\nRules:\n- half_life must be much longer than one job. If not, the usage decays\n inside one run and the mechanism does nothing.\n- A short memory? Use a half_life of 8 to 24 h. Do not use a daily reset:\n a reset makes a cliff, needs a cron job and can be played against.\n- A daily reset is a quota (billing period). That is a different feature.\n- decay is Option. None gives the v5 behaviour, so the\n phase 1 tests do not change.", "originalText": "Setting Scope Set by Changes at runtime\nhalf_life whole scheduler config no (Slurm PriorityDecayHalfLife, 7 d)\nnorm whole scheduler config no\nmultiplier one tenant operator only when the operator changes it\nusage one tenant scheduler at each dispatch and correction\nusage_updated_at one tenant scheduler at each dispatch and correction\n\nRules:\n- half_life must be much longer than one job. If not, the usage decays\n inside one run and the mechanism does nothing.\n- A short memory? Use a half_life of 8 to 24 h. Do not use a daily reset:\n a reset makes a cliff, needs a cron job and can be played against.\n- A daily reset is a quota (billing period). That is a different feature.\n- decay is Option. None gives the v5 behaviour, so the\n phase 1 tests do not change.", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "peOU_G3WMUVh0xNL8En6u", "type": "rectangle", "x": -940.0, "y": -8341.5, "width": 1250, "height": 492.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0026", "roundness": { "type": 3 }, "seed": 786924545, "version": 1, "versionNonce": 1236453726, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false }, { "id": "WtCLbnHrpdZG-BiJwrtmd", "type": "text", "x": -910.0, "y": -8311.5, "width": 261.8, "height": 35.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0024", "roundness": null, "seed": 1703333242, "version": 1, "versionNonce": 467183920, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "Core modules (v6)", "originalText": "Core modules (v6)", "fontSize": 28, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "P18-5P6ywoCrYCT7JyCd0", "type": "text", "x": -910.0, "y": -8261.5, "width": 766.8, "height": 382.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0025", "roundness": null, "seed": 524079123, "version": 1, "versionNonce": 851592884, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "barista-core, phase 1 done (55 tests):\n ids, time, tenant, resource, lifecycle, job, sfq, scheduler\n\nv6 adds fairshare.rs, between tenant and sfq in the dependency order:\n DecayConfig { half_life: Duration, norm: CpuTime }\n UsageMeter { usage: u64, updated_at: Timestamp }\n decay_to(now) -> UsageMeter -- pure\n charge(cost, now) -> UsageMeter -- pure\n half_power(n, d) -> Q16 fixed point of 2^(-n/d)\n\n TenantAccount += usage: UsageMeter\n SchedulerConfig += decay: Option\n Scheduler::dispatch(resource, now): decay -> weight -> tags -> charge\n select(): ranking unchanged (start_tag, tier, size, enqueue_seq)\n\nbarista-store, phase 3: tenants.usage, tenants.usage_updated_at,\n jobs.weight_at_dispatch. The decay uses the database clock, now().", "originalText": "barista-core, phase 1 done (55 tests):\n ids, time, tenant, resource, lifecycle, job, sfq, scheduler\n\nv6 adds fairshare.rs, between tenant and sfq in the dependency order:\n DecayConfig { half_life: Duration, norm: CpuTime }\n UsageMeter { usage: u64, updated_at: Timestamp }\n decay_to(now) -> UsageMeter -- pure\n charge(cost, now) -> UsageMeter -- pure\n half_power(n, d) -> Q16 fixed point of 2^(-n/d)\n\n TenantAccount += usage: UsageMeter\n SchedulerConfig += decay: Option\n Scheduler::dispatch(resource, now): decay -> weight -> tags -> charge\n select(): ranking unchanged (start_tag, tier, size, enqueue_seq)\n\nbarista-store, phase 3: tenants.usage, tenants.usage_updated_at,\n jobs.weight_at_dispatch. The decay uses the database clock, now().", "fontSize": 18, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "27Pj63-bni93hdSUuw9jZ", "type": "text", "x": -4870.0, "y": -8172.0, "width": 554.4000000000001, "height": 20.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0027", "roundness": null, "seed": 1259836706, "version": 1, "versionNonce": 1287599089, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "v6: item 1 RESOLVED. See panel 'Decay model (v6)' to the right.", "originalText": "v6: item 1 RESOLVED. See panel 'Decay model (v6)' to the right.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "odSDGHXUaS1Mw_TecAafd", "type": "text", "x": -4870.0, "y": -9830.0, "width": 906.4000000000001, "height": 20.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0028", "roundness": null, "seed": 1020900455, "version": 1, "versionNonce": 1384280959, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "v6: weight(tier) becomes tier_weight * multiplier * 2^(-usage / norm). See panel 'Updated weight (v6)'.", "originalText": "v6: weight(tier) becomes tier_weight * multiplier * 2^(-usage / norm). See panel 'Updated weight (v6)'.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "n19P8zAVsoBgCI1CT8VtV", "type": "text", "x": -3560.0, "y": -7940.0, "width": 739.2, "height": 20.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0029", "roundness": null, "seed": 1489968546, "version": 1, "versionNonce": 1622800088, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "v6: the weight is read at each dispatch, so a tier change applies from the next tag.", "originalText": "v6: the weight is read at each dispatch, so a tier change applies from the next tag.", "fontSize": 16, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null }, { "id": "JyhzzZf2djkpym9Qx4AbC", "type": "text", "x": -11590.0, "y": -6130.0, "width": 376.20000000000005, "height": 30.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "index": "ba0030", "roundness": null, "seed": 700543515, "version": 1, "versionNonce": 459663415, "isDeleted": false, "boundElements": [], "updated": 1794734098796, "created": 1794734098796, "link": null, "locked": false, "text": "v6: weight = tier * multiplier * 2^(-usage/norm)\nusage decays with a half-life. See 'Updated weight (v6)'.", "originalText": "v6: weight = tier * multiplier * 2^(-usage/norm)\nusage decays with a half-life. See 'Updated weight (v6)'.", "fontSize": 12, "fontFamily": 5, "textAlign": "left", "verticalAlign": "top", "containerId": null, "autoResize": true, "lineHeight": 1.25, "baseFontSize": null, "labelPosition": null } ], "appState": { "gridSize": 20, "gridStep": 5, "gridModeEnabled": false, "viewBackgroundColor": "#ffffff", "lockedMultiSelections": {} }, "files": {} }