.

.gitignore

@@ -2,6 +2,7 @@ sample.txt
 *.html
 *.log
 *.pyc
+*.bak
 current.log
 extended proc nights only.log
 no nights.log

gen.py

@@ -1,6 +1,7 @@
 import datetime
 import os
 import sys
+import time
 from rota.shifts import NoWorkers, RotaBuilder, SingleShift, days
 
 
@@ -12,7 +13,6 @@ sites = (
     "plymouth",
     "taunton",
     "proc",
-    "truro no proc",
 )
 
 from rota.workers import (
@@ -25,19 +25,28 @@ from rota.workers import (
 )
 
 
-suspend_on_finish = False
+suspend_on_finish = True
 solve = True
-time_to_run = 258500
+time_to_run = 508500
+#time_to_run = 2660000
+time_to_run = 400
 # allow = 5000
-ratio = 0.5
+ratio = 0.01
+#ratio = 0.9
 
 
-start_date = datetime.date(2022, 9, 5)
+start_date = datetime.date(2023, 3, 6)
+#start_date = datetime.date(2023, 5, 1)
+#start_date = start_date + datetime.timedelta(weeks=13)
+# start_date = datetime.date(2022, 10, 10)
 
-Rota = RotaBuilder(start_date, weeks_to_rota=12)
+Rota = RotaBuilder(start_date, weeks_to_rota=26, balance_offset_modifier=1)
+#Rota = RotaBuilder(start_date, weeks_to_rota=20, balance_offset_modifier=1)
 Rota.constraint_options["balance_weekends"] = True
-Rota.constraint_options["max_night_frequency"] = 1
+Rota.constraint_options["max_night_frequency"] = 3 # 3
-Rota.constraint_options["max_weekend_frequency"] = 1
+Rota.constraint_options["max_night_frequency_week_exclusions"] = []
+Rota.constraint_options["max_weekend_frequency"] = 2
+Rota.constraint_options["hard_constrain_pair_separation"] = True
 # Rota.constraint_options["avoid_st2_first_month"] = True
 
 Rota.add_shifts(
@@ -46,30 +55,35 @@ Rota.add_shifts(
         name="exeter_twilight",
         length=12.5,
         days=days[:5],
-        balance_offset=3,
+        balance_offset=4,
-        constraint=[{"name": "max_shifts_per_week", "options": 2}],
+        constraint=[
+            {
+                "name": "max_shifts_per_week",
+                "options": 2,
+            }
+        ],
     ),
     SingleShift(
         sites=("truro", "truro twilights"),
         name="truro_twilight",
         length=12.5,
         days=days[:4],
-        balance_offset=3,
+        balance_offset=4,
     ),
     SingleShift(
         sites=("torbay", "torbay twilights"),
         name="torbay_twilight",
         length=12.5,
         days=days[:4],
-        balance_offset=5,
+        balance_offset=4,
         assign_as_block=True,
     ),
     SingleShift(
-        sites=("plymouth", "plymouth twilights"),
+        sites=("plymouth", "plymouth twilights", "plymouth twilights and weekends"),
         name="plymouth_twilight",
         length=12.5,
         days=days[:5],
-        balance_offset=3,
+        balance_offset=4,
         workers_required=1,
         constraint=[{"name": "max_shifts_per_week", "options": 2}],
     ),
@@ -78,21 +92,21 @@ Rota.add_shifts(
         name="weekend_exeter",
         length=12.5,
         days=days[5:],
-        balance_offset=4,
+        balance_offset=3,
         rota_on_nwds=True,
         force_as_block=True,
-        constraint=[{"name": "preclear2"}, {"name": "postclear2"}, {"name": "single_shift_block_per_week"}],
+        constraint=[{"name": "post", "options": 2}, {"name": "pre", "options": 2}],
-        #constraint=[{"name": "preclear2"}, {"name": "postclear2"},],
+        # constraint=[{"name": "pre", "options": 2}, {"name": "post", "options": 2},],
     ),
     SingleShift(
         sites=("truro",),
         name="weekend_truro",
         length=12.5,
         days=days[4:],
-        balance_offset=4,
+        balance_offset=3,
         rota_on_nwds=True,
         force_as_block=True,
-        constraint=[{"name": "preclear2"}, {"name": "postclear2"}],
+        constraint=[{"name": "pre", "options": 2}, {"name": "post", "options": 2}],
         # force_as_block_unless_nwd=True
     ),
     SingleShift(
@@ -100,68 +114,50 @@ Rota.add_shifts(
         name="weekend_torbay",
         length=12.5,
         days=days[4:],
-        balance_offset=4,
+        balance_offset=3,
         rota_on_nwds=True,
         force_as_block=True,
-        constraint=[{"name": "preclear2"}, {"name": "postclear2"}],
+        constraint=[{"name": "pre", "options": 2}, {"name": "post", "options": 2}],
         # force_as_block_unless_nwd=True
     ),
     SingleShift(
-        sites=("plymouth",),
+        sites=("plymouth", "plymouth twilights and weekends"),
         name="weekend_plymouth1",
         length=8,
         days=days[5:],
-        balance_offset=4,
+        balance_offset=2.9,
         workers_required=1,
         rota_on_nwds=True,
         force_as_block=True,
-        constraint=[{"name": "preclear2"}, {"name": "postclear2"}],
+        constraint=[{"name": "pre", "options": 2}, {"name": "post", "options": 2}],
     ),
     SingleShift(
-        sites=("plymouth",),
+        sites=("plymouth","plymouth twilights and weekends"),
         name="weekend_plymouth2",
         length=8,
         days=days[5:],
-        balance_offset=4,
+        balance_offset=2.9,
         workers_required=1,
         rota_on_nwds=True,
         force_as_block=True,
-        constraint=[{"name": "preclear2"}, {"name": "postclear2"}],
+        constraint=[{"name": "pre", "options": 2}, {"name": "post", "options": 2}],
     ),
     SingleShift(
-        sites=("plymouth", "plymouth twilights"),
+        sites=("plymouth", "plymouth twilights","plymouth twilights and weekends"),
         name="plymouth_bank_holidays",
         length=8,
         days=days[5:],
-        balance_offset=2,
+        balance_offset=1,
         workers_required=1,
         rota_on_nwds=True,
         force_as_block=False,
         bank_holidays_only=True,
     ),
     SingleShift(
-        sites=(sites[:-1]),
+        sites=sites,
         name="night_weekday",
         length=12.25,
         days=days[:4],
-        balance_offset=4.9,
-        balance_weighting=1,
-        # hard_constrain_shift=False,
-        workers_required=3,
-        force_as_block=True,
-        rota_on_nwds=True,
-        constraint=[
-            {"name": "night"},
-            {"name": "preclear2"},
-            {"name": "postclear2"},
-            {"name": "require_remote_site_presence_week", "options": ("plymouth", 1)},
-        ],
-    ),
-    SingleShift(
-        sites=(sites[:-1]),
-        name="night_weekend",
-        length=12.25,
-        days=days[4:],
         balance_offset=3.9,
         balance_weighting=1,
         # hard_constrain_shift=False,
@@ -170,18 +166,40 @@ Rota.add_shifts(
         rota_on_nwds=True,
         constraint=[
             {"name": "night"},
-            {"name": "preclear2"},
+            {"name": "pre", "options": 2},
-            {"name": "postclear2"},
+            {"name": "post", "options": 2},
             {"name": "require_remote_site_presence_week", "options": ("plymouth", 1)},
+            {"name": "limit_grade_number", "options": {2: 1}},
+            {"name": "minimum_grade_number", "options": (4, 1)},
+        ],
+    ),
+    SingleShift(
+        sites=(sites),
+        name="night_weekend",
+        length=12.25,
+        days=days[4:],
+        balance_offset=2.9,
+        balance_weighting=1,
+        # hard_constrain_shift=False,
+        workers_required=3,
+        force_as_block=True,
+        rota_on_nwds=True,
+        constraint=[
+            {"name": "night"},
+            {"name": "pre", "options": 2},
+            {"name": "post", "options": 3},
+            {"name": "require_remote_site_presence_week", "options": ("plymouth", 1)},
+            {"name": "limit_grade_number", "options": {2: 1}},
+            {"name": "minimum_grade_number", "options": (4, 1)},
         ],
     ),
 )
 
-Rota.add_grade_constraint_by_week([2], [1, 2, 3, 4], ["night_weekday", "night_weekend"])
+#Rota.add_grade_constraint_by_week([2], [1, 2], ["night_weekday", "night_weekend"])
 
 load_leave = True
 Rota.build_shifts()
-Rota.add_grade_constraint_by_week([2], [1, 2], Rota.get_shift_names())
+#Rota.add_grade_constraint_by_week([2], [1], Rota.get_shift_names())
 
 if load_leave:
     import leave
@@ -195,7 +213,12 @@ if load_leave:
         w = workers[worker]
         site = w["site"]
         grade = w["grade"]
+        try:
             fte = float(w["fte"]) * 100
+        except ValueError:
+            print(f"{worker} has invalid fte: {fte}")
+            raise ValueError
+
         nwd = w["nwd"]
         end_date = w["end_date"]
         start_date = w["start_date"]
@@ -215,7 +238,9 @@ if load_leave:
                 nwd_end_date = Rota.rota_end_date
 
                 if "[" in i:
+                    print("Split nwd", worker_name, i)
                     a, b = i.split("[")[1][:-1].split("-")
+                    print(f"{a=} {b=}")
                     nwd_start_date = datetime.datetime.strptime(a, "%d/%m/%y").date()
                     nwd_end_date = datetime.datetime.strptime(b, "%d/%m/%y").date()
 
@@ -228,7 +253,21 @@ if load_leave:
                 )
         # nwds = [i.strip()[:3].capitalize() for i in nwd.split("/")] if nwd else None
         end_date = end_date if end_date else None
-        oop = oop.split("-") if oop else None
+
+
+        if oop:
+            formatted_oops = []
+            for dates in oop.split(","):
+                s, e = dates.split("-")
+                formatted_oops.append(
+                    {"start_date": datetime.datetime.strptime(s, "%d/%m/%y").date(), "end_date": datetime.datetime.strptime(e, "%d/%m/%y").date()}
+                )
+
+            oop = formatted_oops
+        else:
+            oop = []
+
+        print(f"{worker} OOP {oop}")
 
         previous_shifts = {}
         # if twi:
@@ -260,8 +299,7 @@ if load_leave:
         work_requests = w["requests"]
         site_pref = w["site_pref"]
 
-        Rota.add_worker(
+        w = Worker(
-            Worker(
                 name=worker_name,
                 site=site.lower(),
                 grade=int(grade[2]),
@@ -270,7 +308,7 @@ if load_leave:
                 nwds=nwds,
                 start_date=start_date,
                 end_date=end_date,
-                opp=oop,
+                oop=oop,
                 not_available_to_work=leave_requests,
                 work_requests=work_requests,
                 remote_site=site_pref,
@@ -279,14 +317,25 @@ if load_leave:
                 shift_balance_extra=w["shift_balance_extra"],
                 bank_holiday_extra=w["bank_holiday_extra"],
             )
-        )
 
-Rota.build_workers()
+        print(w)
-Rota.build_model()
+
+        Rota.add_worker(w)
+        
+
+# Rota.build_workers()
+# Rota.build_model()
 
 solver_options = {"ratio": ratio, "seconds": time_to_run, "threads": 10}
 
-Rota.solve_model(options=solver_options)
+start_time = time.time()
+Rota.build_and_solve(solver_options)
+#Rota.solve_shifts_by_block(solver_options, block_length=10)
+#Rota.solve_shifts_individually(solver_options)
+# Rota.solve_model(options=solver_options)
+end_time = time.time()
+
+print(f"Time taken {end_time-start_time}")
 
 # print(Rota.get_worker_details())
 

leave.py

@@ -16,7 +16,12 @@ def load_leave(Rota):
 
         workers = {}
 
-        download = s.get("https://docs.google.com/spreadsheets/d/e/2PACX-1vS49eVS0f86wUJIhXMq7tMPZfAEJZatER09JLu0xnlTjDBkC-54DkAPT3I0JVXDQXNG6AIh_fVeIRS3/pub?gid=0&single=true&output=csv")
+        headers = {
+            "Cache-Control": "no-cache",
+            "Pragma": "no-cache"
+        }
+
+        download = s.get("https://docs.google.com/spreadsheets/d/e/2PACX-1vR18crGluN4wrL9n1sBdZ2u-JsvaLtQ6hr-R5CPUBOaKpL7_hBbhemyPSrFOhDuWxKlrjh-QUM0HrcD/pub?gid=0&single=true&output=csv", headers=headers)
         decoded_content = download.content.decode('utf-8')
 
         reader = csv.reader(decoded_content.splitlines(), delimiter=',')
@@ -100,7 +105,7 @@ def load_leave(Rota):
 
                 elif row_title == "Pair":
                     if lower_item == "":
-                        worker["pair"] = 0
+                        worker["pair"] = None
                     else:
                         worker["pair"] = int(lower_item)
 

output/block_shifts/timetable.css

@@ -0,0 +1,320 @@
+html {
+
+    /* color: red; */
+}
+
+table {
+    /* margin-top: 50px; */
+    text-align: center;
+    /* table-layout : fixed; */
+    /* width:150px */
+    /* width: 200%; */
+    /* overflow-x: auto;
+    display: block;
+    position: relative; */
+    border-collapse: separate;
+    border-spacing: 0;
+
+}
+
+#main-table {
+    padding-top: 100px;
+}
+
+.shift-table {
+    padding-top: 10px;
+}
+
+.shift-table thead {
+    font-weight: bold;
+}
+
+.table-div {
+    /* overflow-x: auto; */
+    width: 80%;
+    overflow-x: scroll;
+    margin-left: 200px;
+    overflow-y: visible;
+    padding: 0;
+}
+
+.table-div th {
+    text-align: center;
+    white-space: nowrap;
+    transform: rotate(90deg) translateX(-100px);
+    position: sticky;
+    top: 0;
+
+}
+
+.table-div th p {
+    margin: 0 -100%;
+    display: inline-block;
+}
+
+.table-div th p:before {
+    content: '';
+    width: 0;
+    padding-top: 110%;
+    /* takes width as reference, + 10% for faking some extra padding */
+    display: inline-block;
+    vertical-align: middle;
+}
+
+.table-div td,
+th {
+    max-width: 10px;
+    padding: 0px;
+}
+
+.table-div tr {
+    padding: 0px;
+}
+
+.table-div tr:hover {
+    background-color: lightblue;
+}
+
+.table-div tr:hover .unavailable {
+
+    background-color: lightgray;
+}
+
+.Sat {
+    /* opacity: 30%; */
+    /* background-color: lightsteelblue; */
+    border-left: solid 1px lightsteelblue;
+}
+
+.Sun {
+    border-right: solid 1px lightsteelblue;
+
+}
+
+.bank-holiday {
+    border-bottom: dotted 1px blue;
+
+}
+
+th.bank-holiday {
+    color: darkblue;
+
+}
+
+/* th.worker {
+    max-width: 100%;
+    width: 200px;
+} */
+
+.plymouth {
+    color: #9169FF;
+}
+
+.exeter {
+    color: #5F60E8;
+}
+
+.truro {
+    color: #0074D9;
+}
+
+.torquay, .torbay {
+    color: #5FA8E8;
+}
+
+.taunton {
+    color: #e85fdf;
+}
+
+
+.unavailable {
+    color: lightgray;
+}
+
+.shift-requested {
+    color: green;
+}
+
+.worker-summary {
+    position: absolute;
+    display: none;
+    overflow-x: visible;
+    width: 100%;
+    text-align: left;
+}
+
+.site-summary {
+    display: inline-flex;
+}
+
+.site-summary .site {
+    border: 1px solid black;
+    padding: 5px;
+    margin: 20px;
+}
+
+.site-name {
+    font-weight: bold;
+}
+
+.worker-summary span {
+    min-width: 160px;
+    display: inline-flex;
+}
+
+.training-days-lost {
+    padding-top: 5px;
+}
+
+.tdl-settings {
+    padding: 2px;
+}
+
+.table-div+pre {
+    display: none;
+}
+
+.shift-breakdown-button+pre {
+    display: none;
+}
+
+.shift-breakdown-button {
+    display: block;
+}
+
+.nwd {
+    color: lightcoral;
+}
+
+.th {}
+
+.table-div th.worker,
+.table-div td.worker {
+    position: absolute;
+    width: 200px;
+    max-width: 200px;
+    left: 0;
+    top: auto;
+    border-top-width: 1px;
+    /*only relevant for first row*/
+    margin-top: -1px;
+    /*compensate for top border*/
+}
+
+.hidden {
+    opacity: 0%;
+}
+
+.tsummary {
+    max-width: auto;
+}
+
+.header th {
+    max-width: 200px;
+
+}
+
+/* [data-night-at-derriford="0"] {
+    border: 1px dotted orange;
+} */
+
+th.site-title {
+    transform: unset;
+}
+
+.name {
+    max-width: 90px;
+    display: inline-flex;
+    white-space: nowrap;
+    overflow: hidden;
+    /*text-overflow: ellipsis;*/
+}
+
+.shift-highlight {
+    background-color: palegreen;
+}
+
+.pair-match {
+    background-color: red;
+}
+
+summary h2 {
+    display: inline;
+}
+
+.remote-site-match.shift-highlight {
+    background-color: yellow;
+}
+
+td.large-shift-diff-pos::before {
+    content: "*";
+    color: red;
+}
+td.large-shift-diff-neg::before {
+    content: "*";
+    color: green;
+}
+
+tr.large-shift-diff-pos td {
+    border-bottom: solid 1px red;
+    border-top: solid 1px red;
+}
+tr.large-shift-diff-neg td {
+    border-bottom: solid 1px green;
+    border-top: solid 1px green;
+}
+
+/* table.transposed {
+    border-collapse: collapse;
+}
+
+table.transposed tr {
+    display: block;
+    float: left;
+}
+
+table.transposed th,
+table.transposed td {
+    display: block;
+    border: 1px solid black;
+    max-width: 100px;
+} */
+table.transposed {
+}
+
+table.transposed tr {
+}
+
+table.transposed th,
+table.transposed td {
+    width: fit-content;
+    max-width: unset;
+    position: relative;
+    border-top: solid 0.5px black;
+    border-right: solid 0.5px black;
+}
+
+table.transposed .Sat {
+    /* opacity: 30%; */
+    /* background-color: lightsteelblue; */
+    border-top: solid 2px lightsteelblue !important;
+}
+
+table.transposed .Sun {
+    border-bottom: solid 2px lightsteelblue !important;
+
+}
+
+table.transposed .bank-holiday {
+    border-bottom: dotted 2px blue !important;
+
+}
+
+table.transposed th.bank-holiday {
+    color: darkblue;
+
+}
+
+.target-assigned.no-colour {
+    background-color: white !important;
+}

output/block_shifts/timetable.js

@@ -0,0 +1,549 @@
+Object.defineProperties(Array.prototype, {
+    count: {
+        value: function (query) {
+            /* 
+               Counts number of occurrences of query in array, an integer >= 0 
+               Uses the javascript == notion of equality.
+            */
+            var count = 0;
+            for (let i = 0; i < this.length; i++)
+                if (this[i] == query)
+                    count++;
+            return count;
+        }
+    }
+});
+
+const mean = arr => arr.reduce((p, c) => p + c, 0) / arr.length;
+
+tables = $(".table-div");
+
+
+function generateExtra() {
+    $(".auto-generated").remove();
+    global_shifts = new Set();
+
+    total_summed_shift_diffs = 0;
+    total_summed_shift_diffs_abs = 0;
+
+    tables.each((n, table) => {
+
+        rows = $(table).find(".worker-row");
+
+        workers = {};
+
+        rows.each((n, tr) => {
+
+
+            worker_td = $(tr).find("td:first");
+
+            worker = worker_td.attr("data-worker");
+
+            shift_tds = $(tr).find(".rota-day");
+            shifts = []
+
+            shift_tds.each((n, td) => {
+                shifts.push($(td).attr("data-shift"));
+            })
+
+            shifts_unique = new Set(shifts);
+            shifts_unique.delete("");
+            global_shifts = new Set([...global_shifts, ...shifts_unique]);
+
+            weekends_worked = 0;
+
+            weeks = shift_tds.length / 7;
+
+            whole_weekends_worked = 0;
+
+            for (var i = 1; i <= weeks; i++) {
+                if ($(tr).find(`[data-week='${i}'][data-day='Sat']`).attr("data-shift") != "" || $(tr).find(`[data-week='${i}'][data-day='Sun']`).attr("data-shift") != "") {
+                    weekends_worked = weekends_worked + 1;
+                }
+                if ($(tr).find(`[data-week='${i}'][data-day='Sat']`).attr("data-shift") != "" && $(tr).find(`[data-week='${i}'][data-day='Sun']`).attr("data-shift") != "") {
+                    whole_weekends_worked = whole_weekends_worked + 1;
+                }
+            }
+
+            shift_counts = {};
+
+            total_shifts = 0;
+            days_lost = 0;
+            shifts_unique.forEach(el => {
+                shift_counts[el] = shifts.count(el);
+                total_shifts = total_shifts + shifts.count(el);
+
+                // why like this?
+                if (el == "night_weekend") {
+                    whole_weekends_worked = whole_weekends_worked - shifts.count(el) / 3
+                }
+
+                if ($("#global-settings").length == 1) {
+
+                    n = $(`#${el}-tdl`).val();
+                    if (!isNaN(n)) {
+                        days_lost = days_lost + shifts.count(el) * n;
+                    }
+
+
+                }
+
+                // // 5 days for weekday nights
+                // if(el == "night_weekday") { days_lost = days_lost + shifts.count(el) / 4 * 5 }
+                // // 4 days for weekend nights
+                // if(el == "night_weekend") {
+                //     days_lost = days_lost + shifts.count(el) / 3 * 4
+                //     //whole_weekends_worked = whole_weekends_worked - shifts.count(el) / 3
+                // }
+                // // 0.5 for twilights
+                // if(el.includes("twilight")) {
+                //     twilight_days_lost = shifts.count(el) / 2
+                //     days_lost = days_lost + twilight_days_lost
+
+                // }
+
+            });
+
+            // Also 1 day for a whole weekend
+            days_lost = days_lost + whole_weekends_worked
+
+            // Highlight non working days
+            nwds = JSON.parse(worker_td.attr("data-nwds"));
+            nwds.forEach(element => {
+                // Restrict to date ranges
+                start_date = new Date(Date.parse(element[1]))
+                end_date = new Date(Date.parse(element[2]))
+                $(shift_tds).filter(`.${element[0]}`).each(
+                    (n, td_el) => {
+                        td_date = new Date(Date.parse(td_el.dataset.date));
+
+                        if (td_date >= start_date && td_date < end_date) {
+                            $(td_el).addClass("nwd");
+                        }
+
+                    }
+
+                )
+            });
+
+            workers[worker] = {
+                "site": worker_td.attr("data-site"),
+                "nwds": worker_td.attr("data-nwds"),
+                "fte": worker_td.attr("data-fte_adj"),
+                "end_date": worker_td.attr("data-end_date"),
+                "shift_diff": worker_td.attr("data-shift-diff"),
+                "shifts": shifts,
+                "shift_types": shifts_unique,
+                "weekends_worked": weekends_worked,
+                "shift_counts": shift_counts,
+                "days_lost": days_lost,
+                //"shifts"
+            }
+
+            sc = JSON.stringify(shift_counts);
+
+
+            bank_holidays = shift_tds.filter(".bank-holiday").filter(function (i) {
+                return $(this).attr('data-shift') != "";
+            });
+
+            bank_holidays_worked = bank_holidays.length;
+
+            shift_diff = JSON.parse(worker_td.attr("data-shift-diff"));
+            summed_shift_diff = Object.values(shift_diff).reduce((a, b) => a + b);
+
+            total_summed_shift_diffs = total_summed_shift_diffs + summed_shift_diff;
+            total_summed_shift_diffs_abs = total_summed_shift_diffs_abs + Math.abs(summed_shift_diff);
+
+            worker_td.get(0).dataset.shiftDiffSummed = summed_shift_diff.toPrecision(2);
+
+
+            worker_td.after(`<div class='worker-summary auto-generated'>
+            <span>Total shifts: ${total_shifts}, </span>
+            <span>Weekends: ${weekends_worked}, </span>
+            <span>Bank holidays: ${bank_holidays_worked}, </span>
+            <span class='shift-diff-span'>Summed shift diff: ${summed_shift_diff.toPrecision(2)}, </span>
+            
+            </div>`)
+
+            if (summed_shift_diff >= 1) {
+                worker_td.addClass("large-shift-diff-pos")
+                $(tr).find(".shift-diff-span").addClass("large-shift-diff-pos")
+            } else if (summed_shift_diff <= -1) {
+                worker_td.addClass("large-shift-diff-neg")
+                $(tr).find(".shift-diff-span").addClass("large-shift-diff-neg")
+
+            }
+
+        })
+
+
+        sites = {};
+        sites_days_lost = {}
+        for (const worker in workers) {
+            w = workers[worker];
+            if (w.fte == 100) {
+                for (let shift of w.shift_types) {
+                    if (!(w.site in sites)) {
+                        sites[w.site] = {};
+
+                    }
+                    if (!(shift in sites[w.site])) {
+                        sites[w.site][shift] = [w.shift_counts[shift]];
+                    } else {
+                        sites[w.site][shift].push(w.shift_counts[shift]);
+                    }
+                }
+                if (!(w.site in sites_days_lost)) {
+                    sites_days_lost[w.site] = [w.days_lost];
+
+                } else {
+                    sites_days_lost[w.site].push(w.days_lost);
+                }
+            }
+        }
+
+        // Calculate site shift summaries
+        // $(table).append("<div class='site-summary auto-generated'>Average shifts for a 100% FTE trainee</div>");
+
+        // for(const site in sites) {
+        //     $(table).find(".site-summary").append(`<div class='site site-${site}'><span class='site-name'>${site}</span></div>`);
+
+        //     for(const shift in sites[site]) {
+        //         average = mean(sites[site][shift])
+        //         $(table).find(`.site-${site}`).append(`<div>${shift}: ${average.toFixed(2)}</div>`);
+        //     }
+
+        //     days_lost_avg = mean(sites_days_lost[site]);
+        //     $(table).find(`.site-${site}`).append(`<div class='training-days-lost'>Training days lost: ${days_lost_avg.toFixed(2)}</div>`);
+        // }
+
+        summary_button = $("<button class='auto-generated'>Toggle Summary</button>").click(() => {
+            $(table).find(".rota-day").toggleClass("hidden");
+            $(table).find(".worker-summary").toggle();
+
+        });
+
+
+        $(table).before($(`<span>Total summed shift diff: ${total_summed_shift_diffs}</span><br/>`));
+        $(table).before($(`<span>Total summed shift diff: ${total_summed_shift_diffs_abs}</span><br/>`));
+        $(table).before(summary_button);
+
+
+
+    });
+    return global_shifts
+}
+
+shifts = generateExtra();
+
+if (shifts.size < 1) {
+    shifts = $("#shifts-container").data("shifts");
+}
+
+oshifts = Array.from(shifts).sort()
+
+$("body").append("<div><button id='show-colour-diff'>Show colour diff</button><table class='tsummary'></table></div>")
+$("#show-colour-diff").on("click", (evt) => {
+    $("table.tsummary").find(`td.target-assigned`).toggleClass("no-colour");
+});
+
+$("table.tsummary").append("<tr class='header'></tr>");
+h = $("table.tsummary tr.header");
+h.append(`<th>Worker</th>`);
+h.append(`<th>Site</th>`);
+h.append(`<th>FTE</th>`);
+h.append(`<th>FTE (adj)</th>`);
+
+
+oshifts.forEach((s) => {
+    h.append(`<th>${s}</th>`);
+
+});
+
+
+
+$(".table-div .worker-row .worker").each((n, tr) => {
+    //worker_td = $(tr).find("td").first();
+    jtr = $(tr);
+    row = $("<tr></tr>");
+    row.append(`<td>${jtr.data("worker")}</td>`)
+    row.append(`<td>${jtr.data("site")}</td>`)
+    row.append(`<td>${jtr.data("fte")}</td>`)
+    row.append(`<td>${parseFloat(jtr.data("fte_adj")).toFixed(2)}</td>`)
+
+    if (jtr.data("shift-diff-summed") >= 1) {
+        row.addClass("large-shift-diff-pos")
+    } else if (jtr.data("shift-diff-summed") <= -1) {
+        row.addClass("large-shift-diff-neg")
+
+    }
+
+    shift_counts = jtr.data("shift-counts")
+    shift_targets = jtr.data("worker-targets")
+
+    oshifts.forEach((s) => {
+        if (s in shift_targets && shift_targets[s] > 0) {
+            if (s in shift_counts) {
+                c = shift_counts[s];
+            } else {
+                c = 0;
+            }
+            row.append(`<td class="target-assigned ${s}" data-assigned=${c} data-target=${shift_targets[s].toFixed(2)} data-diff=${shift_targets[s].toFixed(2)-c}>${c} (${shift_targets[s].toFixed(2)})</td>`)
+
+        } else {
+            row.append(`<td>-</td>`)
+        }
+    })
+
+    $("table.tsummary").append(row)
+})
+
+shift_max_diff_map = {};
+shift_min_diff_map = {};
+
+oshifts.forEach((s) => {
+    shift_max_diff_map[s] = 0;
+    shift_min_diff_map[s] = 0;
+    $("table.tsummary").find(`td.target-assigned.${s}`).each((n, td) => {
+        shift_max_diff_map[s] = Math.max(shift_max_diff_map[s], td.dataset.diff)
+        shift_min_diff_map[s] = Math.min(shift_min_diff_map[s], td.dataset.diff)
+    })
+
+});
+
+oshifts.forEach((shift) => {
+    $("table.tsummary").find(`td.target-assigned.${shift}`).each((n, td) => {
+        diff = td.dataset.diff;
+
+        if (diff > 0) {
+            max_diff = shift_max_diff_map[shift];
+            var h = 120 - Math.floor((max_diff - diff) * 120 / max_diff);
+            var s = diff / max_diff;
+            //var s = 0.5;
+            var v = 1;
+            $(td).css({
+                backgroundColor: hsv2rgb(h, s, 1)
+            })
+        } else {
+            min_diff = shift_min_diff_map[shift]
+
+            var h = Math.floor((min_diff - diff) * 120 / min_diff);
+            var s = diff / min_diff;
+            //var s = 0.5;
+            var v = 1;
+            $(td).css({
+                backgroundColor: hsv2rgb(h, s, 1)
+            })
+
+        }
+        $(td).addClass("no-colour")
+        //s = Math.max(shift_max_diff_map[s], td.dataset.diff)
+        //shift_min_diff_map[s] = Math.min(shift_max_diff_map[s], td.dataset.diff)
+    })
+
+});
+
+oshifts.forEach((shift) => {
+    button = $(`<button data-shift='${shift}'>${shift}</button>`)
+
+    button.on("click", (evt) => {
+        console.log(evt.target.dataset.shift)
+        $("#shift-timetable-div").empty();
+        table = $("<table>");
+        $("table#main-table th.date").each((n, th) => {
+            row = $("<tr>")
+            row.append(`<td>${th.dataset.date}</td>`);
+            $(`table#main-table td[data-date='${th.dataset.date}'][data-shift='${evt.target.dataset.shift}']`).each((n, td) => {
+
+                //row.append(`<td>${}</td>`)
+                row.append($(td).closest("tr").children("td:first").clone())
+
+            });
+
+            table.append(row)
+        });
+        $("#shift-timetable-div").append(table);
+
+    })
+
+    $("#shift-timetable-options").append(button)
+});
+
+//$("table.summary")
+
+// $("body").prepend("<div id='global-settings'>Settings:<br />Customise training days lost per shift (you need to press recalculate for changes to take effect)<form></form></div>")
+
+// shifts.forEach((s) => {
+
+//     disable = false;
+//     if(s.includes("twilight")) {
+//         value = 0.5;
+//     } else if(s == "night_weekend") {
+//         value = 4 / 3;
+//     } else if(s == "night_weekday") {
+//         value = 5 / 4;
+//     } else {
+//         disable = true;
+//         value = 0;
+//     }
+
+//     if(!disable) {
+//         $("#global-settings form").append(`<span class='tdl-settings'><label for="${s}-tdl">${s}</label><input type="number" id="${s}-tdl" name="${s}-tdl" value='${value}'></span>`);
+//     }
+
+
+// })
+// $("#global-settings").append($("<button >Recalculate</button>").click(() => {
+//     generateExtra();
+// }))
+
+// generateExtra();
+
+$(".table-div + pre").each((n, pre) => {
+
+    $(pre).before($("<button class='shift-breakdown-button'>Show shift breakdown</button>").click(() => {
+        $(pre).toggle();
+    }));
+
+})
+
+$("td").hover((e) => {
+    //start hover
+    worker_td = $(e.target).closest('tr').find('td:first')[0]
+    pair = worker_td.dataset.pair;
+    index = $(e.target).index() - 1;
+    if (pair > 0) {
+        $(`td[data-pair=${pair}]`).each((n, el) => {
+            $($(el).closest('tr').find('td').get(index)).addClass("pair-match")
+        })
+    }
+    if (e.target.dataset.shift != undefined && e.target.dataset.shift.length > 0) {
+        $(`td[data-shift=${e.target.dataset.shift}]`).addClass("shift-highlight")
+
+
+        $(`td[data-remote-site='${e.target.dataset.shiftRemoteSite}']`).closest('tr').find(`td[data-shift=${e.target.dataset.shift}]`).addClass("remote-site-match")
+        // if (e.target.dataset.shiftRemoteSite == worker_td.dataset.remoteSite) {
+        //     $(`td[data-shift-remote-site=${e.target.dataset.shiftRemoteSite}]`).addClass("remote-site-match")
+        // }
+    }
+
+}, (e) => {
+
+    //end hover
+    $("td.pair-match").removeClass("pair-match")
+    $("td.shift-highlight").removeClass("shift-highlight")
+    $("td.remote-site-match").removeClass("remote-site-match")
+
+})
+
+$("#rota-table tr td:first-child").each((n, td) => {
+    $(td).click(() => {
+        viewWorker(td);
+    })
+})
+
+function viewWorker(worker) {
+    ds = worker.dataset;
+    dlg = $(`<div class='dialog' title='${ds.worker}'>
+    Site: ${ds.site}</br>
+    FTE: ${ds.fte} (${ds.fte_adj})</br>
+    Start date: ${ds.start_date}</br>
+    End date: ${ds.end_date}</br>
+    Non working days: ${ds.nwds}</br>
+    Remote site: ${ds.remoteSite}</br>
+    Summed shift diff: ${ds.shiftDiffSummed}
+    </div>`)
+    $("body").append(dlg)
+
+    shifts = JSON.parse(ds.shiftCounts)
+    shift_targets = JSON.parse(ds.workerTargets)
+    shift_diffs = JSON.parse(ds.shiftDiff)
+
+    let shift_count_table = document.createElement("table")
+    shift_count_table.classList.add("shift-table")
+    let header_row = shift_count_table.createTHead().insertRow();
+    header_row.insertCell().appendChild(document.createTextNode("Shift"))
+    header_row.insertCell().appendChild(document.createTextNode("Count"))
+    header_row.insertCell().appendChild(document.createTextNode("Target"))
+    header_row.insertCell().appendChild(document.createTextNode("Diffs"))
+
+
+    table_body = shift_count_table.createTBody()
+
+    for (const shift in shifts) {
+        let row = table_body.insertRow();
+        row.insertCell().appendChild(document.createTextNode(shift))
+        row.insertCell().appendChild(document.createTextNode(shifts[shift]))
+        row.insertCell().appendChild(document.createTextNode(shift_targets[shift].toPrecision(2)))
+        row.insertCell().appendChild(document.createTextNode(shift_diffs[shift].toPrecision(2)))
+    }
+
+    dlg.get(0).appendChild(shift_count_table)
+    dlg.dialog()
+
+
+
+}
+
+$("#export-div").append($("#main-table").clone().attr("id", "gen-table").addClass("transposed"))
+
+$("#gen-table").each(function () {
+    var $this = $(this);
+    var newrows = [];
+    //$this.find("th.site-title").parent().remove();
+    $this.find("tr").each(function (rowToColIndex) {
+        $(this).find("td, th").each(function (colToRowIndex) {
+            if (newrows[colToRowIndex] === undefined) {
+                newrows[colToRowIndex] = $("<tr></tr>");
+            }
+            while (newrows[colToRowIndex].find("td, th").length < rowToColIndex) {
+                newrows[colToRowIndex].append($("<td></td>"));
+            }
+            newrows[colToRowIndex].append($(this));
+        });
+    });
+    $this.find("tr").remove();
+    $.each(newrows, function () {
+        $this.append(this);
+    });
+
+    $this.find("td.rota-day").each((n, el) => {
+        $(el).text(el.dataset.shift)
+    });
+});
+
+function hsv2rgb(h, s, v) {
+    // adapted from http://schinckel.net/2012/01/10/hsv-to-rgb-in-javascript/
+    var rgb, i, data = [];
+    if (s === 0) {
+        rgb = [v, v, v];
+    } else {
+        h = h / 60;
+        i = Math.floor(h);
+        data = [v * (1 - s), v * (1 - s * (h - i)), v * (1 - s * (1 - (h - i)))];
+        switch (i) {
+            case 0:
+                rgb = [v, data[2], data[0]];
+                break;
+            case 1:
+                rgb = [data[1], v, data[0]];
+                break;
+            case 2:
+                rgb = [data[0], v, data[2]];
+                break;
+            case 3:
+                rgb = [data[0], data[1], v];
+                break;
+            case 4:
+                rgb = [data[2], data[0], v];
+                break;
+            default:
+                rgb = [v, data[0], data[1]];
+                break;
+        }
+    }
+    return '#' + rgb.map(function (x) {
+        return ("0" + Math.round(x * 255).toString(16)).slice(-2);
+    }).join('');
+};

output/individual_shifts/timetable.css

@@ -0,0 +1 @@
+../timetable.css

output/individual_shifts/timetable.js

@@ -0,0 +1 @@
+../timetable.js

output/timetable.css

@@ -5,7 +5,6 @@ html {
 
 table {
     /* margin-top: 50px; */
-    padding-top: 100px;
     text-align: center;
     /* table-layout : fixed; */
     /* width:150px */
@@ -18,6 +17,10 @@ table {
 
 }
 
+#main-table {
+    padding-top: 100px;
+}
+
 .shift-table {
     padding-top: 10px;
 }
@@ -186,7 +189,7 @@ th.bank-holiday {
 .th {}
 
 .table-div th.worker,
-td.worker {
+.table-div td.worker {
     position: absolute;
     width: 200px;
     max-width: 200px;
@@ -243,10 +246,22 @@ summary h2 {
     background-color: yellow;
 }
 
-.large-shift-diff::before {
+td.large-shift-diff-pos::before {
     content: "*";
     color: red;
+}
+td.large-shift-diff-neg::before {
+    content: "*";
+    color: green;
+}
 
+tr.large-shift-diff-pos td {
+    border-bottom: solid 1px red;
+    border-top: solid 1px red;
+}
+tr.large-shift-diff-neg td {
+    border-bottom: solid 1px green;
+    border-top: solid 1px green;
 }
 
 /* table.transposed {
@@ -299,3 +314,7 @@ table.transposed th.bank-holiday {
     color: darkblue;
 
 }
+
+.target-assigned.no-colour {
+    background-color: white !important;
+}

output/timetable.js

@@ -146,7 +146,6 @@ function generateExtra() {
             bank_holidays = shift_tds.filter(".bank-holiday").filter(function (i) {
                 return $(this).attr('data-shift') != "";
             });
-            console.log("BH", bank_holidays.length)
 
             bank_holidays_worked = bank_holidays.length;
 
@@ -156,6 +155,8 @@ function generateExtra() {
             total_summed_shift_diffs = total_summed_shift_diffs + summed_shift_diff;
             total_summed_shift_diffs_abs = total_summed_shift_diffs_abs + Math.abs(summed_shift_diff);
 
+            worker_td.get(0).dataset.shiftDiffSummed = summed_shift_diff.toPrecision(2);
+
 
             worker_td.after(`<div class='worker-summary auto-generated'>
             <span>Total shifts: ${total_shifts}, </span>
@@ -165,9 +166,13 @@ function generateExtra() {
             
             </div>`)
 
-            if (Math.abs(summed_shift_diff) >= 1) {
+            if (summed_shift_diff >= 1) {
-                worker_td.addClass("large-shift-diff")
+                worker_td.addClass("large-shift-diff-pos")
-                $(tr).find(".shift-diff-span").addClass("large-shift-diff")
+                $(tr).find(".shift-diff-span").addClass("large-shift-diff-pos")
+            } else if (summed_shift_diff <= -1) {
+                worker_td.addClass("large-shift-diff-neg")
+                $(tr).find(".shift-diff-span").addClass("large-shift-diff-neg")
+
             }
 
         })
@@ -176,12 +181,9 @@ function generateExtra() {
         sites = {};
         sites_days_lost = {}
         for (const worker in workers) {
-            //console.log(worker, workers[worker]);
             w = workers[worker];
             if (w.fte == 100) {
-                //console.log(w.shift_types);
                 for (let shift of w.shift_types) {
-                    //console.log(shift);
                     if (!(w.site in sites)) {
                         sites[w.site] = {};
 
@@ -234,16 +236,17 @@ function generateExtra() {
 }
 
 shifts = generateExtra();
-console.log(shifts)
 
 if (shifts.size < 1) {
     shifts = $("#shifts-container").data("shifts");
 }
-console.log(shifts)
 
 oshifts = Array.from(shifts).sort()
 
-$("body").append("<div><table class='tsummary'></table></div>")
+$("body").append("<div><button id='show-colour-diff'>Show colour diff</button><table class='tsummary'></table></div>")
+$("#show-colour-diff").on("click", (evt) => {
+    $("table.tsummary").find(`td.target-assigned`).toggleClass("no-colour");
+});
 
 $("table.tsummary").append("<tr class='header'></tr>");
 h = $("table.tsummary tr.header");
@@ -269,6 +272,13 @@ $(".table-div .worker-row .worker").each((n, tr) => {
     row.append(`<td>${jtr.data("fte")}</td>`)
     row.append(`<td>${parseFloat(jtr.data("fte_adj")).toFixed(2)}</td>`)
 
+    if (jtr.data("shift-diff-summed") >= 1) {
+        row.addClass("large-shift-diff-pos")
+    } else if (jtr.data("shift-diff-summed") <= -1) {
+        row.addClass("large-shift-diff-neg")
+
+    }
+
     shift_counts = jtr.data("shift-counts")
     shift_targets = jtr.data("worker-targets")
 
@@ -279,7 +289,7 @@ $(".table-div .worker-row .worker").each((n, tr) => {
             } else {
                 c = 0;
             }
-            row.append(`<td>${c} (${shift_targets[s].toFixed(2)})</td>`)
+            row.append(`<td class="target-assigned ${s}" data-assigned=${c} data-target=${shift_targets[s].toFixed(2)} data-diff=${shift_targets[s].toFixed(2)-c}>${c} (${shift_targets[s].toFixed(2)})</td>`)
 
         } else {
             row.append(`<td>-</td>`)
@@ -289,6 +299,77 @@ $(".table-div .worker-row .worker").each((n, tr) => {
     $("table.tsummary").append(row)
 })
 
+shift_max_diff_map = {};
+shift_min_diff_map = {};
+
+oshifts.forEach((s) => {
+    shift_max_diff_map[s] = 0;
+    shift_min_diff_map[s] = 0;
+    $("table.tsummary").find(`td.target-assigned.${s}`).each((n, td) => {
+        shift_max_diff_map[s] = Math.max(shift_max_diff_map[s], td.dataset.diff)
+        shift_min_diff_map[s] = Math.min(shift_min_diff_map[s], td.dataset.diff)
+    })
+
+});
+
+oshifts.forEach((shift) => {
+    $("table.tsummary").find(`td.target-assigned.${shift}`).each((n, td) => {
+        diff = td.dataset.diff;
+
+        if (diff > 0) {
+            max_diff = shift_max_diff_map[shift];
+            var h = 120 - Math.floor((max_diff - diff) * 120 / max_diff);
+            var s = diff / max_diff;
+            //var s = 0.5;
+            var v = 1;
+            $(td).css({
+                backgroundColor: hsv2rgb(h, s, 1)
+            })
+        } else {
+            min_diff = shift_min_diff_map[shift]
+
+            var h = Math.floor((min_diff - diff) * 120 / min_diff);
+            var s = diff / min_diff;
+            //var s = 0.5;
+            var v = 1;
+            $(td).css({
+                backgroundColor: hsv2rgb(h, s, 1)
+            })
+
+        }
+        $(td).addClass("no-colour")
+        //s = Math.max(shift_max_diff_map[s], td.dataset.diff)
+        //shift_min_diff_map[s] = Math.min(shift_max_diff_map[s], td.dataset.diff)
+    })
+
+});
+
+oshifts.forEach((shift) => {
+    button = $(`<button data-shift='${shift}'>${shift}</button>`)
+
+    button.on("click", (evt) => {
+        console.log(evt.target.dataset.shift)
+        $("#shift-timetable-div").empty();
+        table = $("<table>");
+        $("table#main-table th.date").each((n, th) => {
+            row = $("<tr>")
+            row.append(`<td>${th.dataset.date}</td>`);
+            $(`table#main-table td[data-date='${th.dataset.date}'][data-shift='${evt.target.dataset.shift}']`).each((n, td) => {
+
+                //row.append(`<td>${}</td>`)
+                row.append($(td).closest("tr").children("td:first").clone())
+
+            });
+
+            table.append(row)
+        });
+        $("#shift-timetable-div").append(table);
+
+    })
+
+    $("#shift-timetable-options").append(button)
+});
+
 //$("table.summary")
 
 // $("body").prepend("<div id='global-settings'>Settings:<br />Customise training days lost per shift (you need to press recalculate for changes to take effect)<form></form></div>")
@@ -340,11 +421,8 @@ $("td").hover((e) => {
     if (e.target.dataset.shift != undefined && e.target.dataset.shift.length > 0) {
         $(`td[data-shift=${e.target.dataset.shift}]`).addClass("shift-highlight")
 
-        console.log(`td[data-remote-site='${e.target.dataset.shiftRemoteSite}']`)
-        console.log($(e.target).closest('tr').find(`td[data-remote-site='${e.target.dataset.shiftRemoteSite}']`))
 
         $(`td[data-remote-site='${e.target.dataset.shiftRemoteSite}']`).closest('tr').find(`td[data-shift=${e.target.dataset.shift}]`).addClass("remote-site-match")
-        // console.log(e.target.dataset.shiftRemoteSite , worker_td.dataset.remoteSite)
         // if (e.target.dataset.shiftRemoteSite == worker_td.dataset.remoteSite) {
         //     $(`td[data-shift-remote-site=${e.target.dataset.shiftRemoteSite}]`).addClass("remote-site-match")
         // }
@@ -373,11 +451,11 @@ function viewWorker(worker) {
     Start date: ${ds.start_date}</br>
     End date: ${ds.end_date}</br>
     Non working days: ${ds.nwds}</br>
-    Remote site: ${ds.remoteSite}
+    Remote site: ${ds.remoteSite}</br>
+    Summed shift diff: ${ds.shiftDiffSummed}
     </div>`)
     $("body").append(dlg)
 
-    console.log(ds.shiftCounts)
     shifts = JSON.parse(ds.shiftCounts)
     shift_targets = JSON.parse(ds.workerTargets)
     shift_diffs = JSON.parse(ds.shiftDiff)
@@ -399,39 +477,73 @@ function viewWorker(worker) {
         row.insertCell().appendChild(document.createTextNode(shifts[shift]))
         row.insertCell().appendChild(document.createTextNode(shift_targets[shift].toPrecision(2)))
         row.insertCell().appendChild(document.createTextNode(shift_diffs[shift].toPrecision(2)))
-        //console.log(shift, shifts[shift]);
     }
 
     dlg.get(0).appendChild(shift_count_table)
     dlg.dialog()
 
 
-    console.log(worker);
 
 }
 
 $("#export-div").append($("#main-table").clone().attr("id", "gen-table").addClass("transposed"))
 
-$("#gen-table").each(function() {
+$("#gen-table").each(function () {
     var $this = $(this);
     var newrows = [];
     //$this.find("th.site-title").parent().remove();
-        $this.find("tr").each(function(rowToColIndex){
+    $this.find("tr").each(function (rowToColIndex) {
-            $(this).find("td, th").each(function(colToRowIndex){
+        $(this).find("td, th").each(function (colToRowIndex) {
-                if(newrows[colToRowIndex] === undefined) { newrows[colToRowIndex] = $("<tr></tr>"); }
+            if (newrows[colToRowIndex] === undefined) {
-                while(newrows[colToRowIndex].find("td, th").length < rowToColIndex){
+                newrows[colToRowIndex] = $("<tr></tr>");
+            }
+            while (newrows[colToRowIndex].find("td, th").length < rowToColIndex) {
                 newrows[colToRowIndex].append($("<td></td>"));
             }
             newrows[colToRowIndex].append($(this));
         });
     });
     $this.find("tr").remove();
-        $.each(newrows, function(){
+    $.each(newrows, function () {
         $this.append(this);
     });
 
     $this.find("td.rota-day").each((n, el) => {
-            console.log(el)
         $(el).text(el.dataset.shift)
     });
-    });
+});
+
+function hsv2rgb(h, s, v) {
+    // adapted from http://schinckel.net/2012/01/10/hsv-to-rgb-in-javascript/
+    var rgb, i, data = [];
+    if (s === 0) {
+        rgb = [v, v, v];
+    } else {
+        h = h / 60;
+        i = Math.floor(h);
+        data = [v * (1 - s), v * (1 - s * (h - i)), v * (1 - s * (1 - (h - i)))];
+        switch (i) {
+            case 0:
+                rgb = [v, data[2], data[0]];
+                break;
+            case 1:
+                rgb = [data[1], v, data[0]];
+                break;
+            case 2:
+                rgb = [data[0], v, data[2]];
+                break;
+            case 3:
+                rgb = [data[0], data[1], v];
+                break;
+            case 4:
+                rgb = [data[2], data[0], v];
+                break;
+            default:
+                rgb = [v, data[0], data[1]];
+                break;
+        }
+    }
+    return '#' + rgb.map(function (x) {
+        return ("0" + Math.round(x * 255).toString(16)).slice(-2);
+    }).join('');
+};

rota/console.py

@@ -0,0 +1,2 @@
+from rich.console import Console
+console = Console()

rota/shifts.py

@@ -1,7 +1,9 @@
 from calendar import week
 import datetime
+from distutils.log import debug
 import itertools
 from typing import Dict, Iterable, List, Sequence, Tuple, Set, Any, Type
+import time
 
 from pydantic import BaseModel, Extra, constr
 
@@ -13,6 +15,7 @@ from pyomo.environ import *
 from pyomo.opt import SolverFactory
 
 from rota.workers import Worker
+from rota.console import console
 
 import json
 
@@ -23,6 +26,19 @@ from collections import defaultdict
 from io import StringIO
 import sys
 
+from rich.pretty import pprint
+from rich.progress import track
+from rich.panel import Panel
+
+import logging
+
+logging.basicConfig(
+    filename="rota.log",
+    filemode="w",
+    format="%(name)s - %(levelname)s - %(message)s",
+    level=logging.DEBUG,
+)
+
 ShiftName = str
 DayStr = str
 WeekInt = int
@@ -36,9 +52,12 @@ from govuk_bank_holidays.bank_holidays import BankHolidays
 
 bank_holidays = BankHolidays()
 bank_holiday_map = {}
+logging.debug("Creating bank holiday map")
 for bank_holiday in bank_holidays.get_holidays(division="england-and-wales"):
     bank_holiday_map[bank_holiday["date"]] = bank_holiday["title"]
 
+logging.debug(bank_holiday_map)
+
 SHIFT_BOUNDS = {
     "bank_holiday": (0, 9),
     "shift_count": (0, 400),
@@ -161,42 +180,19 @@ class RotaBuilder(object):
         SHIFT_BOUNDS=SHIFT_BOUNDS,
     ):
 
-        print(f"Start time: {datetime.datetime.now()}")
+        console.print(Panel(f"""[white]
-        print(f"Weeks to rota: {weeks_to_rota}")
+{locals()}
-        print(f"Use previous shifts {use_previous_shifts}")
+        """, style="green", title="Generating Rota"), justify="left")
-        print(f"Use previous shifts (balance_extra) {use_shift_balance_extra}")
 
         self.SHIFT_BOUNDS = SHIFT_BOUNDS
         self.shifts: List[SingleShift] = []  # type List[SingleShift]
 
-        self.days = days
-
-        self.weeks = [i for i in range(1, weeks_to_rota + 1)]
-
-        self.weeks_days_product = list(itertools.product(self.weeks, days))
-
         self.use_previous_shifts = use_previous_shifts
         self.use_shift_balance_extra = use_shift_balance_extra
         self.use_bank_holiday_extra = use_bank_holiday_extra
 
-        if start_date.weekday() != 0:
-            raise ValueError(
-                f"Start date {start_date.isoformat()} must be a Mon (not a {days[start_date.weekday()]})"
-            )
-
-        self.start_date = start_date
-        self.rota_days_length = len(self.weeks) * 7
-        self.rota_end_date = self.start_date + datetime.timedelta(self.rota_days_length)
-
-        self.unavailable_to_work = set()
-        self.unavailable_to_work_reason = {}
-        self.pref_not_to_work = {}
-        self.pref_not_to_work_reason = {}
-        self.work_requests: set[Tuple[str, WeekInt, DayStr, ShiftName]] = set()
-        self.work_requests_map = {}
 
         self.workers: list[Worker] = []
-        self.worker_pairs = []
 
         # self.night_blocks = ["weekday", "weekend", "none"]
 
@@ -205,7 +201,6 @@ class RotaBuilder(object):
         self.balance_offset_modifier = balance_offset_modifier
         self.ltft_balance_offset = ltft_balance_offset
 
-
         self.constraint_options = {
             "balance_nights": True,
             "constrain_time_off_after_nights": False,
@@ -221,7 +216,8 @@ class RotaBuilder(object):
             # Don't assign multiple shifts every (n) weeks
             "max_weekend_frequency": 1,  # Requires balance weekends
             # Don't assign multiple shifts every (n) weeks
-            "max_night_frequency": 4,
+            "max_night_frequency": 1,
+            "max_night_frequency_week_exclusions": [],
             "max_shifts_per_week": 7,
             "max_shifts_per_month": 40,
             # The following will cause an unsolvable problem if a shift
@@ -235,6 +231,36 @@ class RotaBuilder(object):
             "avoid_shifts_by_grades": [],
         }
 
+
+        self.results = None
+
+        self.ignore_valid_shifts = False
+
+        self.set_rota_dates(start_date, weeks_to_rota)
+
+    def set_rota_dates(self, start_date: datetime.datetime, weeks_to_rota: int):
+
+        self.weeks_to_rota = weeks_to_rota
+
+        self.days = days
+
+        self.weeks = [i for i in range(1, weeks_to_rota + 1)]
+
+        self.weeks_days_product = list(itertools.product(self.weeks, days))
+
+        if start_date.weekday() != 0:
+            raise ValueError(
+                f"Start date {start_date.isoformat()} must be a Mon (not a {days[start_date.weekday()]})"
+            )
+
+        self.start_date = start_date
+        self.rota_days_length = len(self.weeks) * 7
+        self.rota_end_date = self.start_date + datetime.timedelta(self.rota_days_length)
+
+        console.print(
+            f"Weeks to rota: {weeks_to_rota} ({self.start_date} - {self.rota_end_date})"
+        )
+
         # Generate a map for week, day combinations to a given date
         self.week_day_date_map = {}
 
@@ -244,15 +270,32 @@ class RotaBuilder(object):
             self.week_day_date_map[(week, day)] = d
             n = n + 1
 
-        self.results = None
+        self.unavailable_to_work = set()
+        self.unavailable_to_work_reason = {}
+        self.pref_not_to_work = {}
+        self.pref_not_to_work_reason = {}
+        self.work_requests: set[Tuple[str, WeekInt, DayStr, ShiftName]] = set()
+        self.work_requests_map = {}
 
     def solve_model(
-        self, solver: str = "cbc", use_neos: bool = False, options: dict = {}
+        self,
+        solver: str = "cbc",
+        use_neos: bool = False,
+        options: dict = {},
+        debug_if_fail: bool = False,
     ):
-        print("Setting up solver")
+        console.print("Setting up solver")
+
+        solver = "scip"
+
+        if solver == "scip":
+            self.opt = SolverFactory(solver, executable="scip")
+        else:
             self.opt = SolverFactory(solver)
 
-        print("Solving")
+
+
+        console.print("Solving")
         if use_neos:
             solver_manager = SolverManagerFactory("neos")  # Solve using neos server
             # results = solver_manager.solve(Rota.model, opt=opt, logfile="test.log")
@@ -272,20 +315,77 @@ class RotaBuilder(object):
 
         self.results = results
 
-        print(results)
+        console.print(f"Complete - outcome: {results.solver.status}")
-        results.solver.status
+
+        if results.solver.status != "ok" and debug_if_fail:
+            console.print(f"Attempting each shift individually")
+            self.solve_shifts_individually(options)
+
         if not results.solver.status:
             sys.exit(0)
 
+    def solve_shifts_by_block(self, options, block_length: int, folder="block_shifts"):
+        shifts = self.shifts
+
+        outcomes = {}
+
+        no_blocks = self.weeks_to_rota - block_length
+
+        original_start_date = self.start_date
+
+        for block in range(0, no_blocks):
+            start_time = time.time()
+            start_date = original_start_date + datetime.timedelta(weeks=block)
+            self.set_rota_dates(start_date = start_date,weeks_to_rota=block_length)
+            console.print(f"Testing block: start_date - {self.start_date} , length {self.weeks_to_rota} weeks")
+            self.clear_shifts()
+            self.add_shifts(*shifts)
+            self.build_and_solve(options)
+            self.export_rota_to_html(filename=f"{start_date}_{self.weeks_to_rota}", folder=folder)
+            end_time = time.time()
+            time_taken = end_time - start_time
+
+            outcomes[f"{start_date}_{self.weeks}"] = (
+                self.results.solver.status,
+                self.results.solver.termination_condition,
+                time_taken
+            )
+
+        console.print(outcomes)
+        sys.exit(0)
+
+    def solve_shifts_individually(self, options, folder="individual_shifts"):
+        shifts = self.shifts
+
+        outcomes = {}
+
+        for shift in shifts:
+            console.print(f"Testing shifts: {shift.name}")
+            self.ignore_valid_shifts = True
+            self.clear_shifts()
+            self.add_shift(shift)
+            self.build_and_solve(options)
+            self.export_rota_to_html(filename=shift.name, folder=folder)
+
+            outcomes[shift.name] = (
+                self.results.solver.status,
+                self.results.solver.termination_condition,
+            )
+
+        console.print(outcomes)
+
     def build_and_solve(
-        self, options={"ratio": 0.1, "seconds": 1000, "threads": 10}, solve=True
+        self,
+        options={"ratio": 0.1, "seconds": 1000, "threads": 10},
+        solve=True,
+        debug_if_fail: bool = False,
     ):
         self.build_shifts()
         self.build_workers()
         self.build_model()
 
         if solve:
-            self.solve_model(options=options)
+            self.solve_model(options=options, debug_if_fail=debug_if_fail)
 
     def build_model(self):
         # Initialize model
@@ -635,11 +735,10 @@ class RotaBuilder(object):
             else:
                 work_request_sets.add(work_request[3])
 
-
         # Validate shifts are valid
-        #work_request_sets = set([i[3] for i in self.work_requests])
+        # work_request_sets = set([i[3] for i in self.work_requests])
         invalid_shifts = work_request_sets - set(self.get_shift_names())
-        if invalid_shifts:
+        if invalid_shifts and not self.ignore_valid_shifts:
             raise InvalidShift(
                 f"Invalid worker shift request [shift(s): ${invalid_shifts}]"
             )
@@ -911,7 +1010,7 @@ class RotaBuilder(object):
         #                    - self.model.night_per_site_t2[week, block, site]
         #                    == self.model.night_per_site[week, block, site] - 1
         #                )
-        for site in self.sites:
+        for site in track(self.sites, description="Generating site balance constraints..."):
             for shift in self.get_shifts_with_constraint("balance_across_groups"):
                 if site in shift.sites:
                     block = shift.name
@@ -935,7 +1034,7 @@ class RotaBuilder(object):
 
         weeks_days = self.get_week_day_combinations()
 
-        for week in self.weeks:
+        for week in track(self.weeks, description="Generating week constraints..."):
             for shift_name in self.shifts_to_assign_or_force_as_blocks():
                 shift = self.get_shift_by_name(shift_name)
                 for worker in self.get_workers_for_shift(shift):
@@ -1023,6 +1122,8 @@ class RotaBuilder(object):
 
         # Most of our constraints apply per worker
         for worker in self.workers:
+            console.rule(f"Generate worker constraints: [bold blue]{worker.name}[/bold blue]")
+            logging.debug(f"Generate worker constraints: {worker.name}")
 
             if self.constraint_options["minimise_shift_diffs"]:
                 self.model.constraints.add(
@@ -1035,6 +1136,7 @@ class RotaBuilder(object):
 
             for week_blocks in self.get_week_block_iterator(4):
                 # Prevent more than n number shifts per 4 weeks
+                try:
                     self.model.constraints.add(
                         self.constraint_options["max_shifts_per_month"]
                         >= sum(
@@ -1043,12 +1145,20 @@ class RotaBuilder(object):
                             if week in week_blocks
                         )
                     )
+                except ValueError:
+                    # Occurs if there are no shifts within a defined block
+                    # TODO: test if this breaks (and we should check rathar than except)
+                    logging.debug(f"Failed to constrain max_shifts_per_month for worker: {worker.name}")
+                    pass
 
             for constraint_dict in self.constraint_options["avoid_shifts_by_grades"]:
                 if invalid_shifts := set(constraint_dict["shifts"]).difference(
                     set(self.get_shift_names())
                 ):
-                    raise ValueError(
+                    if self.ignore_valid_shifts:
+                        continue  # Skip if we don't want to raise an error
+                    else:
+                        raise InvalidShift(
                             f"avoid shifts by grade constraint contains a non existent shift: {invalid_shifts}"
                         )
                 if invalid_grades := set(constraint_dict["grades"]).difference(
@@ -1103,7 +1213,7 @@ class RotaBuilder(object):
             # Balance shifts within required limits (set by balance_offset)
             # If balance_offset is too restrictive (for the rota length)
             # no solution will be possible
-            for shift in self.get_shifts():
+            for shift in track(self.get_shifts(), description="Generate shift balance constraints"):
                 if (
                     worker.site in shift.sites
                 ):  # Each site specfies which sites self.workers can fullfill it
@@ -1433,9 +1543,15 @@ class RotaBuilder(object):
                                 )
 
             if self.constraint_options["balance_blocks"]:
+                if self.constraint_options["max_night_frequency"]:
                     for week_blocks in self.get_week_block_iterator(
                         self.constraint_options["max_night_frequency"]
                     ):
+                        # Ignore excluded weeks
+                        if set(week_blocks).intersection(set(self.constraint_options["max_night_frequency_week_exclusions"])):
+                            continue
+
+
                         if self.get_shifts_with_constraint("night"):
                             # Prevent nights more than once every n weeks
                             self.model.constraints.add(
@@ -1445,12 +1561,17 @@ class RotaBuilder(object):
                                         shift.name, week, worker.id
                                     ]
                                     for week in week_blocks
-                                for shift in self.get_shifts_with_constraint("night")
+                                    for shift in self.get_shifts_with_constraint(
+                                        "night"
+                                    )
                                 )
                             )
 
             # if self.constraint_options["balance_weekends"]:
-            for week_blocks in self.get_week_block_iterator(self.constraint_options["max_weekend_frequency"]):
+            if self.constraint_options["max_weekend_frequency"]:
+                for week_blocks in self.get_week_block_iterator(
+                    self.constraint_options["max_weekend_frequency"]
+                ):
                     # Prevent weekend shifts more than once every n weeks
                     self.model.constraints.add(
                         1
@@ -1504,7 +1625,7 @@ class RotaBuilder(object):
                         )
                     )
 
-                
+                try:
                     self.model.constraints.add(
                         self.constraint_options["max_shifts_per_week"]
                         >= sum(
@@ -1516,6 +1637,8 @@ class RotaBuilder(object):
                             )
                         )
                     )
+                except ValueError:
+                    pass
 
                 # TODO: consider excluding shifts that span relevant period
                 if (
@@ -1667,7 +1790,7 @@ class RotaBuilder(object):
                 #     self.model.shift_week_worker_assigned[s.name, week, worker.id]
                 #     for s in self.get_shifts_with_constraint("night")))
 
-                #for shift in self.get_shifts_with_constraint("single_block_per_week"):
+                # for shift in self.get_shifts_with_constraint("single_block_per_week"):
                 #    if shift.force_as_block:
                 #        # Force nights to be assigned in blocks
                 #        # self.model.constraints.add(8* self.model.shift_week_worker_assigned[shift.name, week, worker.id] >= sum(self.model.works[worker.id, week, day, shift.name] for day in self.days)
@@ -1709,64 +1832,171 @@ class RotaBuilder(object):
 
                 #
 
-            for n in range(len(weeks_days)):
+            # def get_pre_may(week_days, max_pre)
+
+            for n in track(range(len(weeks_days)), description="Generate week/day constraints"):
 
                 week, day = weeks_days[n]
 
-                p1 = 1
+                pre_map = []
-                p2 = 1
+                for pre_n in range(1, self.max_pre + 1):
-                p3 = 1
+                    try:
-                p4 = 1
+                        pweek, pday = weeks_days[n - pre_n]
-                p5 = 1
+                        p = 1
-                if n > 0:
+                    except:
-                    pweek, pday = weeks_days[n - 1]
-                else:
-                    p1 = 0
                         pweek, pday = weeks_days[n]
+                        p = 0
 
-                try:
+                    pre_map.append((p, pweek, pday))
-                    p2week, p2day = weeks_days[n - 2]
-                except IndexError:
-                    p2week, p2day = weeks_days[n]
-                    p2 = 0
 
+                post_map = []
+                for post_n in range(1, self.max_post + 1):
                     try:
-                    p3week, p3day = weeks_days[n - 3]
+                        pweek, pday = weeks_days[n + post_n]
-                except IndexError:
+                        p = 1
-                    p3week, p3day = weeks_days[n]
+                    except:
-                    p3 = 0
+                        pweek, pday = weeks_days[n]
+                        p = 0
 
-                try:
+                    post_map.append((p, pweek, pday))
-                    p4week, p4day = weeks_days[n - 4]
-                except IndexError:
-                    p4week, p4day = weeks_days[n]
-                    p4 = 0
 
-                try:
+                # p1 = 1
-                    p5week, p5day = weeks_days[n - 5]
+                # p2 = 1
-                except IndexError:
+                # p3 = 1
-                    p5week, p5day = weeks_days[n]
+                # p4 = 1
-                    p5 = 0
+                # p5 = 1
+                # p6 = 1
+                # p7 = 1
+                # if n > 0:
+                #    pweek, pday = weeks_days[n - 1]
+                # else:
+                #    p1 = 0
+                #    pweek, pday = weeks_days[n]
 
-                n1 = 1
+                # try:
-                n2 = 1
+                #    p2week, p2day = weeks_days[n - 2]
-                try:
+                # except IndexError:
-                    nweek, nday = weeks_days[n + 1]
+                #    p2week, p2day = weeks_days[n]
-                except IndexError:
+                #    p2 = 0
-                    nweek, nday = weeks_days[n]
-                    n1 = 0
 
-                try:
+                # try:
-                    n2week, n2day = weeks_days[n + 2]
+                #    p3week, p3day = weeks_days[n - 3]
-                except IndexError:
+                # except IndexError:
-                    n2week, n2day = weeks_days[n]
+                #    p3week, p3day = weeks_days[n]
-                    n2 = 0
+                #    p3 = 0
 
-                try:
+                # try:
-                    n3week, n3day = weeks_days[n + 3]
+                #    p4week, p4day = weeks_days[n - 4]
-                except IndexError:
+                # except IndexError:
-                    n3week, n3day = weeks_days[n]
+                #    p4week, p4day = weeks_days[n]
-                    n3 = 0
+                #    p4 = 0
+
+                # try:
+                #    p5week, p5day = weeks_days[n - 5]
+                # except IndexError:
+                #    p5week, p5day = weeks_days[n]
+                #    p5 = 0
+
+                # try:
+                #    p6week, p6day = weeks_days[n - 6]
+                # except IndexError:
+                #    p6week, p6day = weeks_days[n]
+                #    p6 = 0
+
+                # try:
+                #    p7week, p7day = weeks_days[n - 7]
+                # except IndexError:
+                #    p7week, p7day = weeks_days[n]
+                #    p7 = 0
+
+                # pre_map = [
+                #    (p1, pweek, pday),
+                #    (p2, p2week, p2day),
+                #    (p3, p3week, p3day),
+                #    (p4, p4week, p4day),
+                #    (p5, p5week, p5day),
+                #    (p6, p6week, p6day),
+                #    (p7, p7week, p7day),
+                # ]
+
+                # n1 = 1
+                # n2 = 1
+                # n3 = 1
+                # n4 = 1
+                # n5 = 1
+                # n6 = 1
+                # n7 = 1
+                # if n > 0:
+                #    nweek, nday = weeks_days[n - 1]
+                # else:
+                #    n1 = 0
+                #    nweek, nday = weeks_days[n]
+
+                # try:
+                #    n2week, n2day = weeks_days[n - 2]
+                # except IndexError:
+                #    n2week, n2day = weeks_days[n]
+                #    n2 = 0
+
+                # try:
+                #    n3week, n3day = weeks_days[n - 3]
+                # except IndexError:
+                #    n3week, n3day = weeks_days[n]
+                #    n3 = 0
+
+                # try:
+                #    n4week, n4day = weeks_days[n - 4]
+                # except IndexError:
+                #    n4week, n4day = weeks_days[n]
+                #    n4 = 0
+
+                # try:
+                #    n5week, n5day = weeks_days[n - 5]
+                # except IndexError:
+                #    n5week, n5day = weeks_days[n]
+                #    n5 = 0
+
+                # try:
+                #    n6week, n6day = weeks_days[n - 6]
+                # except IndexError:
+                #    n6week, n6day = weeks_days[n]
+                #    n6 = 0
+
+                # try:
+                #    n7week, n7day = weeks_days[n - 7]
+                # except IndexError:
+                #    n7week, n7day = weeks_days[n]
+                #    n7 = 0
+
+                # post_map = [
+                #    (n1, nweek, nday),
+                #    (n2, n2week, n2day),
+                #    (n3, n3week, n3day),
+                #    (n4, n4week, n4day),
+                #    (n5, n5week, n5day),
+                #    (n6, n6week, n6day),
+                #    (n7, n7week, n7day),
+                # ]
+
+                # n1 = 1
+                # n2 = 1
+                # try:
+                #    nweek, nday = weeks_days[n + 1]
+                # except IndexError:
+                #    nweek, nday = weeks_days[n]
+                #    n1 = 0
+
+                # try:
+                #    n2week, n2day = weeks_days[n + 2]
+                # except IndexError:
+                #    n2week, n2day = weeks_days[n]
+                #    n2 = 0
+
+                # try:
+                #    n3week, n3day = weeks_days[n + 3]
+                # except IndexError:
+                #    n3week, n3day = weeks_days[n]
+                #    n3 = 0
 
                 # IF paired we check the following against both workers
                 workers = [worker]
@@ -1810,8 +2040,9 @@ class RotaBuilder(object):
                                 )
 
                 for constraint_shift in self.get_shifts_with_constraints(
-                    "preclear", "preclear2"
+                    "pre",
                 ):
+                    for n in range(0, constraint_shift.constraint_options["pre"]):
                         if day in constraint_shift.days:
                             self.model.constraints.add(
                                 1
@@ -1819,35 +2050,47 @@ class RotaBuilder(object):
                                     worker.id, week, day, constraint_shift.name
                                 ]
                                 + sum(
-                                p1 * self.model.works[w.id, pweek, pday, shiftname]
+                                    pre_map[n][0]
+                                    * self.model.works[
+                                        w.id, pre_map[n][1], pre_map[n][2], shiftname
+                                    ]
                                     for shiftname in self.get_shift_names_by_week_day(
-                                    pweek, pday
+                                        pre_map[n][1], pre_map[n][2]
                                     )
                                     if shiftname != constraint_shift.name
                                     for w in workers
                                 )
                             )
+                # for constraint_shift in self.get_shifts_with_constraint( "pre"):
+                #    to_sum = []
+                #    for n in range(
+                #        0, constraint_shift.constraint_options["pre"]
+                #    ):
+                #        for w in workers:
+                #            for wk, dy, shiftname in self.get_week_day_shift_names_by_week_day(
+                #                pre_map[n][1], pre_map[n][2]
+                #            ):
+                #                if shiftname != constraint_shift.name:
+                #                    to_sum.append(pre_map[n][0]
+                #                    * self.model.works[
+                #                        w.id, wk, dy, shiftname
+                #                    ])
 
-                for constraint_shift in self.get_shifts_with_constraint("preclear2"):
+                #    if day in constraint_shift.days:
-                    if day in constraint_shift.days:
+                #        self.model.constraints.add(
-                        self.model.constraints.add(
+                #            1
-                            1
+                #            >= self.model.works[
-                            >= self.model.works[
+                #                worker.id, week, day, constraint_shift.name
-                                worker.id, week, day, constraint_shift.name
+                #            ]
-                            ]
+                #            + sum(
-                            + sum(
+                #                to_sum
-                                p2 * self.model.works[w.id, p2week, p2day, shiftname]
+                #            )
-                                for shiftname in self.get_shift_names_by_week_day(
+                #        )
-                                    p2week, p2day
-                                )
-                                if shiftname != constraint_shift.name
-                                for w in workers
-                            )
-                        )
 
                 for constraint_shift in self.get_shifts_with_constraints(
-                    "postclear", "postclear2"
+                    "post",
                 ):
+                    for n in range(0, constraint_shift.constraint_options["post"]):
                         if day in constraint_shift.days:
                             self.model.constraints.add(
                                 1
@@ -1855,51 +2098,117 @@ class RotaBuilder(object):
                                     worker.id, week, day, constraint_shift.name
                                 ]
                                 + sum(
-                                n1 * self.model.works[w.id, nweek, nday, shiftname]
+                                    post_map[n][0]
+                                    * self.model.works[
+                                        w.id, post_map[n][1], post_map[n][2], shiftname
+                                    ]
                                     for shiftname in self.get_shift_names_by_week_day(
-                                    nweek, nday
+                                        post_map[n][1], post_map[n][2]
                                     )
                                     if shiftname != constraint_shift.name
                                     for w in workers
                                 )
                             )
 
-                for constraint_shift in self.get_shifts_with_constraints("postclear2"):
+                #                for constraint_shift in self.get_shifts_with_constraints(
-                    if day in constraint_shift.days:
+                #                    "preclear", "preclear2"
-                        self.model.constraints.add(
+                #                ):
-                            1
+                #                    if day in constraint_shift.days:
-                            >= self.model.works[
+                #                        self.model.constraints.add(
-                                worker.id, week, day, constraint_shift.name
+                #                            1
-                            ]
+                #                            >= self.model.works[
-                            + sum(
+                #                                worker.id, week, day, constraint_shift.name
-                                n2 * self.model.works[w.id, n2week, n2day, shiftname]
+                #                            ]
-                                for shiftname in self.get_shift_names_by_week_day(
+                #                            + sum(
-                                    n2week, n2day
+                #                                p1 * self.model.works[w.id, pweek, pday, shiftname]
-                                )
+                #                                for shiftname in self.get_shift_names_by_week_day(
-                                if shiftname != constraint_shift.name
+                #                                    pweek, pday
-                                for w in workers
+                #                                )
-                            )
+                #                                if shiftname != constraint_shift.name
-                        )
+                #                                for w in workers
+                #                            )
+                #                        )
+                #
+                #                for constraint_shift in self.get_shifts_with_constraint("preclear2"):
+                #                    if day in constraint_shift.days:
+                #                        self.model.constraints.add(
+                #                            1
+                #                            >= self.model.works[
+                #                                worker.id, week, day, constraint_shift.name
+                #                            ]
+                #                            + sum(
+                #                                p2 * self.model.works[w.id, p2week, p2day, shiftname]
+                #                                for shiftname in self.get_shift_names_by_week_day(
+                #                                    p2week, p2day
+                #                                )
+                #                                if shiftname != constraint_shift.name
+                #                                for w in workers
+                #                            )
+                #                        )
+
+                # for constraint_shift in self.get_shifts_with_constraints(
+                #    "postclear", "postclear2"
+                # ):
+                #    if day in constraint_shift.days:
+                #        self.model.constraints.add(
+                #            1
+                #            >= self.model.works[
+                #                worker.id, week, day, constraint_shift.name
+                #            ]
+                #            + sum(
+                #                n1 * self.model.works[w.id, nweek, nday, shiftname]
+                #                for shiftname in self.get_shift_names_by_week_day(
+                #                    nweek, nday
+                #                )
+                #                if shiftname != constraint_shift.name
+                #                for w in workers
+                #            )
+                #        )
+
+                # for constraint_shift in self.get_shifts_with_constraints("postclear2"):
+                #    if day in constraint_shift.days:
+                #        self.model.constraints.add(
+                #            1
+                #            >= self.model.works[
+                #                worker.id, week, day, constraint_shift.name
+                #            ]
+                #            + sum(
+                #                n2 * self.model.works[w.id, n2week, n2day, shiftname]
+                #                for shiftname in self.get_shift_names_by_week_day(
+                #                    n2week, n2day
+                #                )
+                #                if shiftname != constraint_shift.name
+                #                for w in workers
+                #            )
+                #        )
 
                 # Night constraint means we won't assign a shift the day before
                 # an unavailability
                 for constraint_shift in self.get_shifts_with_constraint("night"):
                     if (
                         worker.id,
-                        pweek,
+                        # pweek,
-                        pday,
+                        pre_map[0][1],
+                        # pday,
+                        pre_map[0][2],
                         constraint_shift.name,
                     ) in self.model.works:
                         # Ensure night prior to unavalibity is not assigned
                         self.model.constraints.add(
                             self.model.available[worker.id, week, day]
                             >= self.model.works[
-                                worker.id, pweek, pday, constraint_shift.name
+                                # worker.id, pweek, pday, constraint_shift.name
+                                worker.id,
+                                pre_map[0][1],
+                                pre_map[0][2],
+                                constraint_shift.name,
                             ]
                         )
 
         self.define_objectives()
 
+        print("Building model completed")
+
     def define_objectives(self):
         # Define an objective function with model as input, to pass later
         def obj_rule(m):
@@ -2106,7 +2415,7 @@ class RotaBuilder(object):
         self.workers_id_map = {}
         self.workers_name_map = {}
 
-        for worker in self.workers:
+        for worker in track(self.workers, description="Building workers"):
             worker.load_rota(self)
             wid = worker.id
             if wid in self.workers_id_map:
@@ -2125,15 +2434,16 @@ class RotaBuilder(object):
         self.full_time_equivalent_sites = {}
         self.workers_at_sites = {}
 
-        for site in self.sites:
+        for site in track(self.sites, description="Generating site workers"):
             self.workers_at_sites[site] = [w for w in self.workers if w.site == site]
             self.full_time_equivalent_sites[site] = sum(
                 w.fte_adj for w in self.workers if w.site == site
             )
 
+        self.worker_pairs = []
         pairs = defaultdict(list)
         for w in self.workers:
-            if w.pair > 0:
+            if w.pair is not None:
                 pairs[w.pair].append(w)
 
         for p in pairs:
@@ -2154,7 +2464,7 @@ class RotaBuilder(object):
         """
         self.shifts.append(shift)
 
-    def add_shifts(self, *shifts: SingleShift) -> None:
+    def add_shifts(self, *shifts: SingleShift ) -> None:
         """Add multiple shifts
 
         Returns:
@@ -2162,6 +2472,9 @@ class RotaBuilder(object):
         """
         self.shifts.extend(shifts)
 
+    def clear_shifts(self) -> None:
+        self.shifts = []
+
     def get_shift_names_by_week_day(self, week, day: DayStr) -> set:
         """Returns the shifts required for a specific day
 
@@ -2171,6 +2484,18 @@ class RotaBuilder(object):
         """
         return self.week_day_shifts_dict[(week, day)]
 
+    def get_week_day_shift_names_by_week_day(self, week, day: DayStr) -> set:
+        """Returns the shifts required for a specific day
+
+        Returns:
+            set: Set of ShiftName
+
+        """
+        return set(
+            (week, day, shift_name)
+            for shift_name in self.week_day_shifts_dict[(week, day)]
+        )
+
     def get_shift_by_name(self, name: str) -> SingleShift:
         """
 
@@ -2240,6 +2565,14 @@ class RotaBuilder(object):
                         self.shift_counts[s.name] = self.shift_counts[s.name] + 1
         # print(self.week_day_shift_product)
 
+        self.max_pre = 1
+        self.max_post = 1
+        for shift in self.get_shifts_with_constraint("pre"):
+            self.max_pre = max(shift.constraint_options["pre"], self.max_pre)
+
+        for shift in self.get_shifts_with_constraint("post"):
+            self.max_post = max(shift.constraint_options["post"], self.max_post)
+
     #        # Todo replace with week_day.....
     #        self.day_shift_product = []
     #        self.day_shiftclass_product = []
@@ -2366,7 +2699,7 @@ class RotaBuilder(object):
         """ """ """
         return self.shift_names
 
-    def get_week_block_iterator(self, block_length):
+    def get_week_block_iterator(self, block_length: int):
         """Gets a two dimensional list of week blocks in specified length
 
         e.g. block_length = 4
@@ -2496,8 +2829,14 @@ class RotaBuilder(object):
         return self.start_date + datetime.timedelta(weeks=week)
 
     # RESULTS
-    def export_rota_to_html(self, filename: str = "rota"):
+    def export_rota_to_html(self, filename: str = "rota", folder=None):
+        if folder is not None:
+            output_file = Path("output", folder, f"{filename}.html")
+        else:
             output_file = Path("output", f"{filename}.html")
+
+        # Make sure path exits
+        output_file.parent.mkdir(parents=True, exist_ok=True)
         with open(output_file, "w") as f:
             f.write(self.get_worker_timetable_html(True))
 
@@ -2639,9 +2978,9 @@ class RotaBuilder(object):
         for week, day in self.get_week_day_combinations():
             d = self.start_date + datetime.timedelta(n)
 
-            th_class = ""
+            th_class = "date"
             if d in bank_holiday_map:
-                th_class = "bank-holiday"
+                th_class = th_class + " bank-holiday"
 
             date_row.append(
                 f"<th title='{d}' class='{th_class}' data-date='{d}'>Week {week}: {day}</th>"
@@ -2859,6 +3198,13 @@ class RotaBuilder(object):
         <div id="export-div">
         <div>
         </details>
+        <details>
+        <summary><h2>Shift timetables</h2></summary>
+        <div id="shift-timetable-options">
+        </div>
+        <div id="shift-timetable-div">
+        <div>
+        </details>
         </div
         </body>
         """

rota/workers.py

@@ -4,6 +4,7 @@ from typing import Iterable, List, Literal, Optional
 from pydantic import BaseModel, Extra, validator
 
 from rich.pretty import pprint
+from rota.console import console
 
 # from .shifts import RotaBuilder, days, sites
 import uuid
@@ -80,7 +81,7 @@ class Worker(BaseModel):
     previous_shifts: dict = {}
     shift_balance_extra: dict = {}
     bank_holiday_extra: int = 0
-    pair: int = 0
+    pair: str | None = None
 
     class Config:
         extra = Extra.allow
@@ -120,12 +121,16 @@ class Worker(BaseModel):
 
         if self.name == "Steph bailey":
             print(self.start_date, self.end_date)
+            print(self.oop)
 
         # if no start date default to the start of the rota
         if self.start_date is None:
             self.calculated_start_date = Rota.start_date
         else:
             # ? test if start date is valid
+            if self.start_date < Rota.start_date:
+                self.calculated_start_date = self.start_date
+            else:
                 self.calculated_start_date = self.start_date
 
         if self.end_date is None:
@@ -186,6 +191,7 @@ class Worker(BaseModel):
                         start_oop_date = Rota.start_date
 
                     oop_length = (end_oop_date - start_oop_date).days
+                    print(f"oop length {oop_length}")
                     days_to_work = days_to_work - oop_length
 
                     days_until_oop = (start_oop_date - Rota.start_date).days
@@ -199,9 +205,13 @@ class Worker(BaseModel):
                             (self.id, week, day)
                         ] = f"OOP ({oop_name})".format(self.oop)
 
+        days_to_end = (self.calculated_end_date - Rota.start_date).days
+
         # loop throught dates converting to week / day combination
         for item in self.pref_not_to_work:
             days_from_start = (item.date - Rota.start_date).days
+            # Ignore dates past the end of the rota (or end date)
+            if days_from_start < days_to_end:
                 week = days_from_start // 7 + 1
                 day = Rota.days[(days_from_start % 7)]
                 # Weight the value to take into account the number of preferences
@@ -214,12 +224,18 @@ class Worker(BaseModel):
 
         # print(not_available_to_work)
         # loop throught dates converting to week / day combination
+
+        unavailable_set = set()
         for item in self.not_available_to_work:
             days_from_start = (item.date - Rota.start_date).days
+            # Ignore dates past the end of the rota (or end date)
+            if days_from_start < days_to_end:
                 week = days_from_start // 7 + 1
                 day = Rota.days[(days_from_start % 7)]
                 Rota.unavailable_to_work.add((self.id, week, day))
                 Rota.unavailable_to_work_reason[(self.id, week, day)] = item.reason
+                unavailable_set.add((week, day))
+
 
         for item in self.work_requests:
             days_from_start = (item.date - Rota.start_date).days
@@ -257,6 +273,10 @@ class Worker(BaseModel):
         if days_to_work < 1:
             self.fte_adj = 0
 
+        if days_to_work > 0 and (days_to_work - len(unavailable_set)) / days_to_work < 0.3:
+            console.print(f"{ self.name }: {days_to_work}, {Rota.rota_days_length}")
+            console.print(f"Warning {self.name} has excessive unavailability [{len(unavailable_set)}/{days_to_work}, adjusted fte={self.fte_adj}] (this may be infeasible)", style="red")
+
     def __lt__(self, other) -> bool:
         return (self.site, self.grade, self.fte_adj, self.name) < (
             other.site,
@@ -266,18 +286,19 @@ class Worker(BaseModel):
         )
 
     def __str__(self) -> str:
+        return f"{self.name}, {self.site}, {self.oop}"
 
-        nwds = (
+        #nwds = (
-            ", ".join([str(i) for i, start, end in self.non_working_day_list])
+        #    ", ".join([str(i) for i, start, end in self.non_working_day_list])
-            if self.non_working_day_list is not None
+        #    if self.non_working_day_list is not None
-            else ""
+        #    else ""
-        )
+        #)
-        return "{} {} [{}] REMOTE SITE:{}".format(
+        #return "{} {} [{}] REMOTE SITE:{}".format(
-            self.name,
+        #    self.name,
-            (self.site, self.grade, self.fte_adj),
+        #    (self.site, self.grade, self.fte_adj),
-            nwds,
+        #    nwds,
-            self.remote_site,
+        #    self.remote_site,
-        )
+        #)
 
     def get_details(self) -> str:
         return "{} {} {}".format(self.id, self.site[0], self.grade)

scip.set

@@ -0,0 +1,7 @@
+parallel/minnthreads = 5
+parallel/maxnthreads = 10
+
+limits/solutions = -1
+limits/gap = 0
+
+limits/time = 20000

test/rota_constraints/test_avoid_grades_by_week.py

@@ -85,7 +85,7 @@ class TestWorkerRequests:
                 force_as_block=False,
             ),
         )
-        with pytest.raises(ValueError):
+        with pytest.raises(InvalidShift):
 
             Rota.add_grade_constraint_by_week([1], [1,2,3], ["b"])
             Rota.build_and_solve(options={"ratio": 0.000})

test/test_nights2.py

@@ -0,0 +1,512 @@
+import datetime
+import pytest
+from rota.shifts import InvalidShift, NoWorkers, RotaBuilder, SingleShift, days
+
+from rota.workers import Worker
+
+
+def generate_basic_rota(weeks_to_rota=10):
+    start_date = datetime.date(2022, 3, 7)
+
+    Rota = RotaBuilder(
+        start_date,
+        weeks_to_rota=weeks_to_rota,
+    )
+
+    # Add a few workers
+    Rota.add_workers(
+        [
+            Worker(name="worker1", site="group1", grade=1),
+            Worker(name="worker2", site="group1", grade=1),
+        ]
+    )
+
+    return Rota
+
+
+def date_generator(from_date, days):
+    n = 0
+    while True:
+        yield from_date
+
+        n = n + 1
+        if n >= days:
+            break
+        from_date = from_date + datetime.timedelta(days=1)
+
+
+class TestNightShifts:
+    def test_nights(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}, {"name": "pre", "options": 2}],
+                workers_required=2,
+            ),
+            SingleShift(
+                sites=("group1", "group2"),
+                name="b",
+                length=12.5,
+                days=days[:3],
+                force_as_block=False,
+                # constraint=[{"name": "night"}],
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "ok"
+        assert Rota.results.solver.termination_condition == "optimal"
+
+    def test_nights_pre3(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+                workers_required=2,
+            ),
+            SingleShift(
+                sites=("group1", "group2"),
+                name="b",
+                length=12.5,
+                days=days[:2],
+                force_as_block=False,
+                # constraint=[{"name": "night"}],
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night_pre3")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_nights_pre3_2(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+                workers_required=1,
+            ),
+            SingleShift(
+                sites=("group1", "group2"),
+                name="b",
+                length=12.5,
+                days=days[:3],
+                force_as_block=False,
+                # constraint=[{"name": "night"}],
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night_pre3")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_nights_fail(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+                workers_required=2,
+            ),
+            SingleShift(
+                sites=("group1", "group2"),
+                name="b",
+                length=12.5,
+                days=days[:3],
+                force_as_block=False,
+                # constraint=[{"name": "night"}],
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+
+        assert Rota.results.solver.status == "warning"
+
+    def test_nights2(self):
+        Rota = generate_basic_rota()
+
+        Rota.constraint_options["max_night_frequency"] = 0
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}],
+                workers_required=2,
+            ),
+            SingleShift(
+                sites=("group1", "group2"),
+                name="m",
+                length=12.5,
+                days=days[:5],
+                force_as_block=True,
+                # constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+                # constraint=[{"name": "night"}],
+                workers_required=2,
+            ),
+            # SingleShift(
+            #    sites=("group1", "group2"), name="b", length= 12.5, days=days[:3],
+            #    force_as_block=False,
+            #    #constraint=[{"name": "night"}],
+            # ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_nights_pre_wrap_around(self):
+        Rota = generate_basic_rota(weeks_to_rota=8)
+
+        Rota.constraint_options["max_night_frequency"] = 0
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[:2],
+                force_as_block=True,
+                constraint=[
+                    {"name": "night"},
+                    {"name": "pre", "options": 5},
+                    {"name": "post", "options": 5},
+                ],
+                workers_required=3,
+            ),
+            # SingleShift(
+            #    sites=("group1", "group2"), name="m", length= 12.5, days=days[:5],
+            #    force_as_block=True,
+            #    #constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+            #    #constraint=[{"name": "night"}],
+            #    workers_required=2,
+            # ),
+            # SingleShift(
+            #    sites=("group1", "group2"), name="b", length= 12.5, days=days[:3],
+            #    force_as_block=False,
+            #    #constraint=[{"name": "night"}],
+            # ),
+        )
+
+        Rota.add_workers(
+            [
+                Worker(
+                    name="worker3",
+                    site="group1",
+                    grade=1,
+                    not_available_to_work=[
+                        {
+                            "date": Rota.rota_end_date - datetime.timedelta(days=1),
+                            "reason": "no wanna",
+                        },
+                        # {
+                        #    "date": Rota.rota_end_date - datetime.timedelta(days=5),
+                        #    "reason": "no wanna",
+                        # },
+                    ],
+                ),
+            ]
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_nights_max_frequency(self):
+        Rota = generate_basic_rota()
+
+        Rota.constraint_options["max_night_frequency"] = 1
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}],
+                workers_required=2,
+            ),
+            # SingleShift(
+            #    sites=("group1", "group2"),
+            #    name="m",
+            #    length=12.5,
+            #    days=days[:5],
+            #    force_as_block=True,
+            #    # constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+            #    # constraint=[{"name": "night"}],
+            #    workers_required=2,
+            # ),
+            # SingleShift(
+            #    sites=("group1", "group2"), name="b", length= 12.5, days=days[:3],
+            #    force_as_block=False,
+            #    #constraint=[{"name": "night"}],
+            # ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_nights_max_frequency_fail(self):
+        Rota = generate_basic_rota()
+
+        Rota.constraint_options["max_night_frequency"] = 2
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}],
+                workers_required=2,
+            ),
+            # SingleShift(
+            #    sites=("group1", "group2"),
+            #    name="m",
+            #    length=12.5,
+            #    days=days[:5],
+            #    force_as_block=True,
+            #    # constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+            #    # constraint=[{"name": "night"}],
+            #    workers_required=2,
+            # ),
+            # SingleShift(
+            #    sites=("group1", "group2"), name="b", length= 12.5, days=days[:3],
+            #    force_as_block=False,
+            #    #constraint=[{"name": "night"}],
+            # ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "warning"
+
+    def test_nights_max_frequency3(self):
+        Rota = generate_basic_rota()
+
+        Rota.constraint_options["max_night_frequency"] = 2
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}],
+                workers_required=1,
+            ),
+            SingleShift(
+                sites=("group1", "group2"),
+                name="m",
+                length=12.5,
+                days=days[:5],
+                force_as_block=True,
+                # constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+                 constraint=[{"name": "night"}],
+                workers_required=1,
+            ),
+            # SingleShift(
+            #    sites=("group1", "group2"), name="b", length= 12.5, days=days[:3],
+            #    force_as_block=False,
+            #    #constraint=[{"name": "night"}],
+            # ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "warning"
+
+    def test_nights_max_frequency4(self):
+        Rota = generate_basic_rota()
+
+        Rota.constraint_options["max_night_frequency"] = 2
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}],
+                workers_required=1,
+            ),
+            SingleShift(
+                sites=("group1", "group2"),
+                name="m",
+                length=12.5,
+                days=days[:5],
+                force_as_block=True,
+                # constraint=[{"name": "night"}, {"name": "pre", "options": 3}],
+                constraint=[{"name": "night"}],
+                workers_required=1,
+            ),
+            # SingleShift(
+            #    sites=("group1", "group2"), name="b", length= 12.5, days=days[:3],
+            #    force_as_block=False,
+            #    #constraint=[{"name": "night"}],
+            # ),
+        )
+
+        Rota.add_workers(
+            [
+                Worker(
+                    name="worker3",
+                    site="group1",
+                    grade=1,
+                ),
+                Worker(
+                    name="worker4",
+                    site="group1",
+                    grade=1,
+                ),
+            ]
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_nights_max_frequency_exclusions(self):
+        Rota = generate_basic_rota()
+
+        Rota.constraint_options["max_night_frequency"] = 2
+        Rota.constraint_options["max_night_frequency_week_exclusions"] = list(range(10))
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}],
+                workers_required=2,
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_nights_max_frequency_exclusions2(self):
+        Rota = generate_basic_rota(weeks_to_rota=8)
+
+        Rota.constraint_options["max_night_frequency"] = 2
+        Rota.constraint_options["max_night_frequency_week_exclusions"] = list(range(1,4))
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                constraint=[{"name": "night"}],
+                workers_required=2,
+            ),
+        )
+
+        Rota.add_workers(
+            [
+                Worker(
+                    name="worker3",
+                    site="group1",
+                    grade=1,
+                    start_date=Rota.start_date + datetime.timedelta(weeks=4)
+                ),
+                Worker(
+                    name="worker4",
+                    site="group1",
+                    grade=1,
+                    start_date=Rota.start_date + datetime.timedelta(weeks=4)
+                ),
+            ]
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_nights_max_frequency_exclusions3(self):
+        # TODO: find out the issues with odd rotas and exclusions
+        Rota = generate_basic_rota(weeks_to_rota=9)
+
+        Rota.constraint_options["max_night_frequency"] = 2
+        Rota.constraint_options["max_night_frequency_week_exclusions"] = list(range(1,5))
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"),
+                name="n",
+                length=12.5,
+                days=days[5:],
+                force_as_block=True,
+                balance_offset=10,
+                constraint=[{"name": "night"}],
+                workers_required=2,
+            ),
+        )
+
+        Rota.add_workers(
+            [
+                Worker(
+                    name="worker3",
+                    site="group1",
+                    grade=1,
+                    start_date=Rota.start_date + datetime.timedelta(weeks=4)
+                ),
+                Worker(
+                    name="worker4",
+                    site="group1",
+                    grade=1,
+                    start_date=Rota.start_date + datetime.timedelta(weeks=4)
+                ),
+            ]
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("night")
+
+        assert Rota.results.solver.status == "ok"

test/test_rota.py

@@ -203,6 +203,7 @@ class TestDemoRotaNights:
     Rota.constraint_options["balance_nights"] = False
     Rota.constraint_options["minimise_shift_diffs"] = False
     Rota.constraint_options["balance_blocks"] = False
+    Rota.constraint_options["max_night_frequency"] = 0
     # Rota.constraint_options["balance_weekends"] = False
 
     # Add a few workers
@@ -227,7 +228,7 @@ class TestDemoRotaNights:
             length=12.5,
             days=days[:4],
             workers_required=1,
-            constraint=[{"name": "preclear2"}, {"name": "postclear2"}],
+            constraint=[{"name": "pre", "options": 1}, {"name": "post", "options": 2}],
             # constraint=["night"],
             force_as_block=True
             # We use a different balance weighting for each shift as otherwise
@@ -240,7 +241,7 @@ class TestDemoRotaNights:
             length=12.5,
             days=days[4:],
             # balance_weighting=0.5
-            constraint=[{"name": "preclear2"}, {"name": "postclear2"}],
+            constraint=[{"name": "pre", "options": 1}, {"name": "post", "options": 2}],
             # constraint=["night"],
             force_as_block=True,
         ),
@@ -274,6 +275,8 @@ class TestDemoRotaNights:
         assert self.Rota.start_date == self.start_date
 
     def test_night_assignment(self):
+        assert self.Rota.results.solver.status == "ok"
+
         shift_summary = self.Rota.get_shift_summary_dict()
 
         for worker_name in shift_summary:
@@ -316,7 +319,7 @@ class TestDemoRotaClear:
         (worker1, worker2, worker3, worker4, worker5, worker6, worker7, worker8)
     )
 
-    def test_preclear(self):
+    def test_pre(self):
         self.Rota.add_shifts(
             SingleShift(
                 sites=("group1", "group2"),
@@ -325,7 +328,7 @@ class TestDemoRotaClear:
                 days=days[:5],
                 balance_offset=40,
                 workers_required=2,
-                constraint=[{"name": "preclear"}],
+                constraint=[{"name": "pre", "options": 1}],
             ),
             SingleShift(
                 sites=("group1", "group2"),
@@ -342,7 +345,7 @@ class TestDemoRotaClear:
                 days=days[3],
                 balance_offset=40,
                 workers_required=1,
-                constraint=[{"name": "preclear2"}, {"name": "postclear2"}],
+                constraint=[{"name": "pre", "options": 1}, {"name": "post", "options": 1}],
             ),
         )
 
@@ -350,7 +353,7 @@ class TestDemoRotaClear:
         self.Rota.build_workers()
         self.Rota.build_model()
 
-        solver_options = {"ratio": 0.1, "seconds": 1000, "threads": 10}
+        solver_options = {"seconds": 1000, "threads": 10}
 
         self.Rota.solve_model(options=solver_options)
         self.Rota.export_rota_to_html("test4")
@@ -369,12 +372,72 @@ class TestDemoRotaClear:
             assert "ca" not in shifts_string
             assert "ac" not in shifts_string
             assert "ca" not in shifts_string
-            assert "a-c" not in shifts_string
+            #assert "a-c" not in shifts_string
-            assert "c-a" not in shifts_string
+            #assert "c-a" not in shifts_string
             assert "bc" not in shifts_string
             assert "cb" not in shifts_string
-            assert "b-c" not in shifts_string
+            #assert "b-c" not in shifts_string
-            assert "c-b" not in shifts_string
+            #assert "c-b" not in shifts_string
+
+    #def test_preclear(self):
+    #    self.Rota.add_shifts(
+    #        SingleShift(
+    #            sites=("group1", "group2"),
+    #            name="a",
+    #            length=12.5,
+    #            days=days[:5],
+    #            balance_offset=40,
+    #            workers_required=2,
+    #            constraint=[{"name": "preclear"}],
+    #        ),
+    #        SingleShift(
+    #            sites=("group1", "group2"),
+    #            name="b",
+    #            length=12.5,
+    #            days=days[:5],
+    #            balance_offset=40,
+    #            workers_required=2,
+    #        ),
+    #        SingleShift(
+    #            sites=("group1", "group2"),
+    #            name="c",
+    #            length=12.5,
+    #            days=days[3],
+    #            balance_offset=40,
+    #            workers_required=1,
+    #            constraint=[{"name": "pre", "options": 2}, {"name": "post", "options": 2}],
+    #        ),
+    #    )
+
+    #    self.Rota.build_shifts()
+    #    self.Rota.build_workers()
+    #    self.Rota.build_model()
+
+    #    solver_options = {"ratio": 0.1, "seconds": 1000, "threads": 10}
+
+    #    self.Rota.solve_model(options=solver_options)
+    #    self.Rota.export_rota_to_html("test4")
+
+    #    assert self.Rota.results.solver.status == "ok"
+    #    assert self.Rota.results.solver.termination_condition == "optimal"
+
+    #    for worker in self.Rota.workers:
+    #        shifts = self.Rota.get_worker_shift_list(worker)
+
+    #        # Convert shift to a string representation
+    #        shifts_string = "".join([i if i != "" else "-" for i in shifts])
+
+    #        # check for patterns that should not occur
+    #        assert "ba" not in shifts_string
+    #        assert "ca" not in shifts_string
+    #        assert "ac" not in shifts_string
+    #        assert "ca" not in shifts_string
+    #        assert "a-c" not in shifts_string
+    #        assert "c-a" not in shifts_string
+    #        assert "bc" not in shifts_string
+    #        assert "cb" not in shifts_string
+    #        assert "b-c" not in shifts_string
+    #        assert "c-b" not in shifts_string
 
 
 class TestDemoRotaShiftConstraints:
@@ -619,7 +682,7 @@ class TestLimitConstraints:
         # Rota.constraint_options["balance_nights_across_sites"] = False
         # Rota.constraint_options["balance_shifts_over_workers"] = False
         #Rota.constraint_options["balance_shifts_quadratic"] = True
-        Rota.build_and_solve(options={"ratio": 0.1})
+        Rota.build_and_solve(options={})
 
         Rota.export_rota_to_html("constraint_limit_grades")
 
@@ -1045,7 +1108,7 @@ class TestNightUnavailable:
 
     Rota.add_workers((worker1, worker2))
 
-    def test_basic_assignment(self, capsys):
+    def test_basic_assignment(self):
         self.Rota.shifts = []
 
         # Add a weekday and weekend shift
@@ -1074,7 +1137,6 @@ class TestNightUnavailable:
     def test_assign_night_prior_to_unavailablity(self):
         self.Rota.shifts = []
 
-        # Add a weekday and weekend shift
         self.Rota.add_shifts(
             SingleShift(
                 sites =("group1",),
@@ -1087,10 +1149,30 @@ class TestNightUnavailable:
         )
 
         self.Rota.build_and_solve(options={"ratio": 0.00})
+        self.Rota.export_rota_to_html("testnight_unavail")
 
         assert self.Rota.results.solver.status == "warning"
         assert self.Rota.results.solver.termination_condition == "infeasible"
 
+    def test_assign_non_night_prior_to_unavailablity(self):
+        self.Rota.shifts = []
+
+        self.Rota.add_shifts(
+            SingleShift(
+                sites =("group1",),
+                name = "night_weekend",
+                length = 12.5,
+                days = days[5:],
+                workers_required=2,
+                #constraint=[{"name": "night"}],
+            ),
+        )
+
+        self.Rota.build_and_solve(options={"ratio": 0.00})
+        self.Rota.export_rota_to_html("testnight_unavail")
+
+        assert self.Rota.results.solver.status == "ok"
+
     def test_assign_prior_to_unavailablity_non_night(self):
         self.Rota.shifts = []
 

test/test_shifts.py

@@ -173,3 +173,142 @@ class TestShiftConstraints:
         Rota.export_rota_to_html("max_shifts_per_week_block")
 
         assert Rota.results.solver.status == "warning"
+
+    def test_pre_shift_constraint(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"), name="a", length= 12.5, days=days[0],
+                constraint=[{"name": "pre", "options": 1}],
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="b", length= 12.5, days=days[1],
+                constraint=[{"name": "pre", "options": 1}],
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="c", length= 12.5, days=days[2],
+                constraint=[{"name": "pre", "options": 1}],
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="d", length= 12.5, days=days[3],
+                constraint=[{"name": "pre", "options": 1}],
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="e", length= 12.5, days=days[4],
+                constraint=[{"name": "pre", "options": 1}],
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="f", length= 12.5, days=days[5],
+                constraint=[{"name": "pre", "options": 1}],
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("pre")
+
+        assert Rota.results.solver.status == "ok"
+
+        for worker in Rota.get_workers():
+            shift_string = Rota.get_worker_shift_list_string(worker)
+
+            print(shift_string)
+            for s in ["ab", "bc", "cd", "de", "ef", "fg"]:
+                assert s not in shift_string
+
+            
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"), name="g", length= 12.5, days=days[6],
+                constraint=[{"name": "pre", "options": 2}],
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+
+        assert Rota.results.solver.status == "warning"
+
+    def test_pre_shift_constraint2(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"), name="a", length= 12.5, days=days[0],
+                constraint=[{"name": "pre", "options": 2}],
+                workers_required=2
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="b", length= 12.5, days=days[4],
+                constraint=[{"name": "pre", "options": 3}],
+                workers_required=2
+            ),
+
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("pre")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_pre_shift_constraint3(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"), name="a", length= 12.5, days=days[0],
+                constraint=[{"name": "pre", "options": 3}],
+                workers_required=2
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="b", length= 12.5, days=days[4],
+                constraint=[{"name": "pre", "options": 3}],
+                workers_required=2
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("pre")
+
+        assert Rota.results.solver.status == "warning"
+
+    def test_post_shift_constraint(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"), name="a", length= 12.5, days=days[0],
+                constraint=[{"name": "post", "options": 3}],
+                workers_required=2
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="b", length= 12.5, days=days[4],
+                constraint=[{"name": "post", "options": 2}],
+                workers_required=2
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("post")
+
+        assert Rota.results.solver.status == "ok"
+
+    def test_post_shift_constraint2(self):
+        Rota = generate_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1", "group2"), name="a", length= 12.5, days=days[0],
+                constraint=[{"name": "post", "options": 3}],
+                workers_required=2
+            ),
+            SingleShift(
+                sites=("group1", "group2"), name="b", length= 12.5, days=days[4],
+                constraint=[{"name": "post", "options": 3}],
+                workers_required=2
+            ),
+        )
+
+        Rota.build_and_solve(options={"ratio": 0.000})
+        Rota.export_rota_to_html("post")
+
+        assert Rota.results.solver.status == "warning"

test/test_workers.py

@@ -318,3 +318,76 @@ class TestWorkers:
         assert Rota.results.solver.termination_condition == "optimal"
 
         Rota.export_rota_to_html("worker_oops")
+
+    def test_worker_pairs(self):
+        Rota = setup_basic_rota()
+
+        Rota.add_shifts(
+            SingleShift(
+                sites=("group1",),
+                name="a",
+                length=12.5,
+                days=days[:5],
+                balance_offset=20,
+                workers_required=1,
+            ),
+            SingleShift(
+                sites=("group1",),
+                name="b",
+                length=12.5,
+                days=days[:5],
+                balance_offset=20,
+                workers_required=1,
+            ),
+        )
+
+        Rota.add_workers(
+            [
+                Worker(name="worker1", site="group1", grade=1, fte=100, pair="1"),
+                Worker(name="worker2", site="group1", grade=1, fte=100, pair="1"),
+            ]
+        )
+
+        Rota.constraint_options["hard_constrain_pair_separation"] = True
+
+        Rota.build_and_solve()
+
+        # With two paired workers we shouldn't be able to solve
+        assert Rota.results.solver.status == "warning"
+
+        worker3 = Worker(name="worker3", site="group1", grade=1, fte=100)
+        Rota.add_worker(worker3)
+
+        # An additional worker can take half the shifts
+        Rota.build_and_solve()
+
+        assert Rota.results.solver.status == "ok"
+
+        assert Rota.get_workers_total_shifts()["worker3"] == 50
+        assert (
+            Rota.get_workers_total_shifts()["worker1"]
+            + Rota.get_workers_total_shifts()["worker2"]
+            == 50
+        )
+
+        worker3.pair = "1"
+
+        Rota.build_and_solve()
+
+        assert Rota.results.solver.status == "warning"
+
+        worker3.pair = 0
+
+        Rota.build_and_solve()
+        Rota.export_rota_to_html("worker_pairs")
+
+        assert Rota.results.solver.status == "ok"
+
+        worker4 = Worker(name="worker4", site="group1", grade=1, fte=100, pair="1")
+        Rota.add_worker(worker4)
+
+        Rota.build_and_solve()
+        Rota.export_rota_to_html("worker_pairs")
+
+        assert Rota.results.solver.status == "ok"
+        assert Rota.get_workers_total_shifts()["worker3"] == 50