Live updates during a cycle; Filter & sort dropdown on phones; layout polish

- Worker publishes the rows swept so far every few seconds (LivePublishSeconds)
  with a DataVersion stamp; the dashboard reloads on each new version, so
  tables, the Overview and open dialogs update mid-cycle. Shared files are
  written to a temp file and swapped in, so readers never see half a file.
- Phones / tablets: one Filter & sort button opens a column list instead of a
  row of chips, on every table.
- Details: a wrapped Simple / Advanced toggle sits on the left.
- Card fields: fixed-height dialog; field list and add list share it evenly.

Config/usmt-dash.psd1

@@ -57,6 +57,10 @@
         # is pinged every cycle).
         # Cycles only run in business hours plus the midnight pitstop: 60 = hourly, plus midnight.
         PresenceSweepMinutes = 60
+
+        # While a cycle runs, the rows swept so far are published this often, so the
+        # dashboard updates live instead of only when the cycle finishes
+        LivePublishSeconds = 10
     }
 
     # -------------------------------------------------------------------------------------
@@ -128,6 +132,7 @@
         DelayMsPerDevice  = 35      # artificial per-device latency so the progress bar moves
         LifecycleHours    = 24      # how long one simulated migration takes, start to finish
         PresenceSweepMinutes = 5    # ping every device this often (Live default is hourly)
+        LivePublishSeconds = 4      # publish swept rows this often during a cycle
         Verbose           = $true   # run the worker with -v style tracing
     }
 

README.md

@@ -73,8 +73,9 @@ a 1,300-device scope.
   after an app are linked from that app in the software view.
 - **Business-hours scheduling**: the worker cycles every few minutes during working hours,
   sleeps overnight, and makes a midnight "pitstop" for the daily software refresh.
-  *Force Update* and *Force Full Refresh* skip the wait. Newly tracked devices are swept and
-  published first; the sync dialog shows how long each kind of device (SCCM association,
+  *Force Update* and *Force Full Refresh* skip the wait. The dashboard updates live while a cycle runs
+  (swept rows are published every few seconds), newly tracked devices are swept and
+  published first, the sync dialog shows how long each kind of device (SCCM association,
   manual pair, completed snapshot...) took.
 
 | | |

USMT-Worker.ps1

@@ -31,6 +31,10 @@ $loadLogName       = if ($config.SCCM.LoadStateLog) { $config.SCCM.LoadStateLog
 # How often every tracked device (cached and destination devices included) is pinged to keep
 # "last online" current. Cycles only run in business hours plus the midnight pitstop, so this
 # works out to an hourly presence sweep during the day and one at midnight.
+# While a cycle runs, rows are published this often (merged with last cycle's rows for devices
+# not swept yet), so the dashboard updates live instead of only when the cycle ends
+$livePublishSeconds = if ($config.IsDemo) { $config.Demo.LivePublishSeconds } else { $config.Worker.LivePublishSeconds }
+if (-not $livePublishSeconds) { $livePublishSeconds = 10 }
 $presenceMinutes   = if ($config.IsDemo) { $config.Demo.PresenceSweepMinutes } else { $config.Worker.PresenceSweepMinutes }
 if (-not $presenceMinutes) { $presenceMinutes = 60 }
 
@@ -50,6 +54,7 @@ $globalLastUpdated = "Never"
 $globalNextRun = "Calculating..."
 $globalLastDuration = "N/A"
 $global:LastBreakdown = $null
+$global:DataVersion = ""
 $global:LastPresenceSweep = [datetime]::MinValue
 
 # --- FUNCTIONS ---
@@ -64,9 +69,10 @@ function Update-WorkerState {
             NextCycle = $nextRun
             LastDuration = $duration
             LastCycle = $global:LastBreakdown
+            DataVersion = $global:DataVersion
         } | ConvertTo-Json -Compress -Depth 5
 
-        [System.IO.File]::WriteAllText($statePath, $json, [System.Text.Encoding]::UTF8)
+        Write-SharedText $statePath $json
     } catch {}
 }
 
@@ -91,6 +97,21 @@ function Set-LogInfoLine {
 
 function Get-IsoNow { return (Get-Date).ToString("o") }
 
+# Files the dashboard reads while the worker writes them: write a temp file, then swap it in,
+# so a reader never sees a half-written file. Falls back to a direct write if the swap fails
+# (e.g. Windows refusing to replace a file another process has open).
+function Write-SharedText([string]$path, [string]$text) {
+    $tmp = "$path.tmp"
+    try {
+        [System.IO.File]::WriteAllText($tmp, $text, [System.Text.Encoding]::UTF8)
+        if (Test-Path -LiteralPath $path) { [System.IO.File]::Replace($tmp, $path, $null) }
+        else { [System.IO.File]::Move($tmp, $path) }
+    } catch {
+        [System.IO.File]::WriteAllText($path, $text, [System.Text.Encoding]::UTF8)
+        if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue }
+    }
+}
+
 function ConvertTo-Hashtable($obj) {
     $h = @{}
     if ($obj -is [hashtable]) { foreach ($k in $obj.Keys) { $h[$k] = $obj[$k] } }
@@ -111,8 +132,7 @@ function Read-JsonTable([string]$path) {
 }
 
 function Write-JsonFile([string]$path, $obj, [int]$depth = 6) {
-    $json = ConvertTo-Json -InputObject $obj -Depth $depth -Compress
-    [System.IO.File]::WriteAllText($path, $json, [System.Text.Encoding]::UTF8)
+    Write-SharedText $path (ConvertTo-Json -InputObject $obj -Depth $depth -Compress)
 }
 
 # --- PROVIDER (data source adapter) ---
@@ -216,6 +236,14 @@ function Use-Snapshot {
     }
 }
 
+# Data.json + a version stamp the dashboard compares to know there's something new to load
+function Publish-Data {
+    param($rows)
+    $global:DataVersion = (Get-Date).ToString("o")
+    $payload = [PSCustomObject]@{ LastUpdated = (Get-Date).ToString("MM/dd/yy hh:mm tt"); DataVersion = $global:DataVersion; Devices = $rows }
+    Write-SharedText $jsonPath ($payload | ConvertTo-Json -Depth 6)
+}
+
 # Every row has the same columns; these are filled in as the device is processed
 function New-DeviceRow {
     param([string]$sName, [string]$dName, [string]$tags, [string]$pairing)
@@ -499,6 +527,7 @@ while ($true) {
     if ($newSources.Count -gt 0) { Write-Host "--- Commencing Data Sweep ($totalDevices Devices, $($newSources.Count) new first) ---" -ForegroundColor White }
     else { Write-Host "--- Commencing Data Sweep ($totalDevices Devices) ---" -ForegroundColor White }
 
+    $publishWatch = [System.Diagnostics.Stopwatch]::StartNew()
     foreach ($deviceName in $sourceDevices) {
         $currentDeviceCount++
         $deviceWatch = [System.Diagnostics.Stopwatch]::StartNew()
@@ -722,14 +751,19 @@ while ($true) {
         $timing[$category].Count++
         $timing[$category].Ms += $deviceWatch.Elapsed.TotalMilliseconds
 
-        # Newly tracked devices are done: publish them now rather than at the end of the cycle
-        if ($newSources.Count -gt 0 -and $knownSources.Count -gt 0 -and $currentDeviceCount -eq $newSources.Count) {
-            Write-Host "   -> [PRIORITY] $($newSources.Count) new device(s) swept. Publishing early..." -ForegroundColor Green
+        # Live updates: newly tracked devices are published as soon as they're done, then the
+        # rows swept so far every few seconds (devices not reached yet keep last cycle's row)
+        $newDone = ($newSources.Count -gt 0 -and $currentDeviceCount -eq $newSources.Count)
+        $remaining = $totalDevices - $currentDeviceCount
+        if ($remaining -gt 0 -and ($newDone -or $publishWatch.Elapsed.TotalSeconds -ge $livePublishSeconds)) {
+            if ($newDone) { Write-Host "   -> [PRIORITY] $($newSources.Count) new device(s) swept. Publishing early..." -ForegroundColor Green }
+            elseif ($v) { Write-Host "   -> [LIVE] Publishing $currentDeviceCount of $totalDevices swept device(s)..." -ForegroundColor DarkGray }
             $interim = [System.Collections.Generic.List[object]]::new()
             foreach ($r in $deviceTable) { $interim.Add($r) }
-            foreach ($dev in $knownSources) { if ($previousRows.ContainsKey($dev)) { $interim.Add($previousRows[$dev]) } }
+            foreach ($dev in $sourceDevices) { if (-not $processedSources.Contains($dev) -and $previousRows.ContainsKey($dev)) { $interim.Add($previousRows[$dev]) } }
             Save-AdvancedCache
-            Write-JsonFile $jsonPath ([PSCustomObject]@{ LastUpdated = (Get-Date).ToString("MM/dd/yy hh:mm tt"); Devices = $interim })
+            Publish-Data $interim
+            $publishWatch.Restart()
         }
     }
     $sweepSeconds = $phaseWatch.Elapsed.TotalSeconds
@@ -806,10 +840,7 @@ while ($true) {
 
     if ($v) { Write-Host "   -> Writing payload to Data.json..." -ForegroundColor DarkGray }
     $globalLastUpdated = (Get-Date).ToString("MM/dd/yy hh:mm tt")
-    $exportPayload = [PSCustomObject]@{ LastUpdated = $globalLastUpdated; Devices = $deviceTable }
-
-    $jsonFinal = $exportPayload | ConvertTo-Json -Depth 6
-    [System.IO.File]::WriteAllText($jsonPath, $jsonFinal, [System.Text.Encoding]::UTF8)
+    Publish-Data $deviceTable
     $saveSeconds = $phaseWatch.Elapsed.TotalSeconds
 
     $cycleEnd = Get-Date

web/demo/demo.js

@@ -721,6 +721,8 @@
         // ---- worker data ----
         'GET Worker-State.json': async () => {
             const state = (await serverGet('/Worker-State.json')) || {};
+            // the visitor's own (simulated) changes count as new data too
+            if (state.DataVersion && Store.state.appliedTracked) state.DataVersion += '#' + Store.state.rev;
             if (Sim.worker) return jsonResponse(Object.assign({}, state, Sim.worker));
             return jsonResponse(state);
         },
@@ -731,7 +733,7 @@
             // the dashboard shows its "Waiting for Worker..." placeholders
             if (!s.appliedTracked || !snap.rows.length) return jsonResponse(snap.data);
             const { rows } = buildRows(s.appliedTracked, snap.rows, snap.adv);
-            return jsonResponse({ LastUpdated: snap.data.LastUpdated || psStamp(), Devices: rows });
+            return jsonResponse({ LastUpdated: snap.data.LastUpdated || psStamp(), DataVersion: (snap.data.DataVersion || '') + '#' + s.rev, Devices: rows });
         },
         'GET Advanced-Cache.json': async () => {
             const s = Store.state;

web/excel-filter.js

@@ -318,35 +318,71 @@
             bar.className = 'xf-bar';
             table.parentNode.insertBefore(bar, table);
         }
+        // One "Filter & sort" button (plus Card fields / Clear) instead of a chip per column:
+        // the column list opens as a sheet, so the bar stays a single short row
+        const sortedCol = cols.find(c => { const d = sortDirOf(ad, c.idx); return d === 'asc' || d === 'desc'; });
         const parts = [];
         if (ad.custom) parts.push('<button type="button" class="xf-chip xf-fields" data-act="fields">&#9776; Card fields</button>');
-        cols.forEach(c => {
-            if (!c.filterable && !c.sortable) return;
-            const on = activeKeys.has(ad.colKey(c.idx));
-            const n = on ? filters[ad.key][ad.colKey(c.idx)].size : 0;
-            const dir = sortDirOf(ad, c.idx);
-            const arrow = dir === 'asc' ? ' &uarr;' : dir === 'desc' ? ' &darr;' : '';
-            parts.push(`<button type="button" class="xf-chip${on ? ' xf-on' : ''}" data-idx="${c.idx}">${on ? FUNNEL_SVG : ''}<span></span>${on ? ` <b>${n}</b>` : ''}${arrow}${CARET_SVG}</button>`);
-        });
-        if (active.length) parts.push('<button type="button" class="xf-chip xf-clear" data-act="clear">Clear all</button>');
+        parts.push(`<button type="button" class="xf-chip xf-filterbtn${active.length ? ' xf-on' : ''}" data-act="pick">${FUNNEL_SVG}<span>Filter &amp; sort</span>${active.length ? ` <b>${active.length}</b>` : ''}${sortedCol ? ' &middot; ' + (sortDirOf(ad, sortedCol.idx) === 'asc' ? '&uarr;' : '&darr;') : ''}${CARET_SVG}</button>`);
+        if (active.length) parts.push('<button type="button" class="xf-chip xf-clear" data-act="clear">Clear</button>');
         const sig = JSON.stringify([parts, cols.map(c => c.label)]);
         if (bar.dataset.sig !== sig) {
             bar.innerHTML = parts.join('');
-            // labels via textContent (column names can come from user-chosen fields)
-            bar.querySelectorAll('.xf-chip[data-idx]').forEach(chip => {
-                chip.querySelector('span').textContent = cols[+chip.dataset.idx].label;
-            });
             bar.dataset.sig = sig;
             bar.onclick = e => {
                 const chip = e.target.closest('.xf-chip');
                 if (!chip) return;
                 if (chip.dataset.act === 'clear') { delete filters[ad.key]; ad.rerun(); return; }
                 if (chip.dataset.act === 'fields') { openFieldEditor(); return; }
-                openMenu(table, +chip.dataset.idx, chip);
+                if (chip.dataset.act === 'pick') openColumnPicker(table, chip);
             };
         }
     }
 
+    // Phones / tablets: pick a column, then its usual sort + filter menu opens
+    function openColumnPicker(table, anchor) {
+        closeMenu();
+        const ad = adapter(table);
+        if (!ad) return;
+        const cols = columnsOf(table, ad).filter(c => c.filterable || c.sortable);
+        const activeKeys = new Set(activeFilters(ad, columnsOf(table, ad)).map(c => ad.colKey(c.idx)));
+        backdrop = el('div', 'xf-backdrop'); backdrop.onclick = closeMenu;
+        menu = el('div', 'xf-menu xf-sheet xf-picker');
+        menuCtx = { picker: true, table };
+        const head = el('div', 'xf-head');
+        head.appendChild(el('span', 'xf-title', 'Filter & sort'));
+        const x = el('button', 'xf-x', '\u00d7'); x.type = 'button'; x.title = 'Close'; x.onclick = closeMenu;
+        head.appendChild(x);
+        menu.appendChild(head);
+        const list = el('div', 'xf-pick-list');
+        cols.forEach(c => {
+            const b = el('button', 'xf-pick-col'); b.type = 'button';
+            b.appendChild(el('span', 'xf-pick-name', c.label));
+            const bits = [];
+            const on = activeKeys.has(ad.colKey(c.idx));
+            if (on) { const f = filters[ad.key][ad.colKey(c.idx)]; bits.push(f.size + ' selected'); }
+            const d = sortDirOf(ad, c.idx);
+            if (d === 'asc') bits.push('sorted \u2191'); else if (d === 'desc') bits.push('sorted \u2193');
+            const st = el('span', 'xf-pick-state', bits.join(' \u00b7 '));
+            if (on) b.classList.add('xf-on');
+            b.appendChild(st);
+            b.insertAdjacentHTML('beforeend', '<svg class="xf-pick-chev" viewBox="0 0 16 16" aria-hidden="true"><path d="M6 3.5 10.5 8 6 12.5" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>');
+            b.onclick = () => { closeMenu(); openMenu(table, c.idx, anchor); };
+            list.appendChild(b);
+        });
+        menu.appendChild(list);
+        if (activeKeys.size) {
+            const actions = el('div', 'xf-actions');
+            const clr = el('button', 'xf-cancel', 'Clear all filters'); clr.type = 'button';
+            clr.onclick = () => { delete filters[ad.key]; closeMenu(); ad.rerun(); };
+            actions.appendChild(clr);
+            menu.appendChild(actions);
+        }
+        document.body.appendChild(backdrop);
+        document.body.appendChild(menu);
+        document.addEventListener('keydown', onMenuKey, true);
+    }
+
     // Effective sort direction of a column: a size/free-space sort overrides the table's own
     function sortDirOf(ad, i) {
         const ns = nativeSort[ad.key];

web/index.html

@@ -897,6 +897,7 @@
         }
         
         let globalDataLastUpdated = "Never";
+        let globalDataVersion = "";
         let globalTrackedHash = "";
 
         const groupsArray = ["No Data", "Not Started", "Backup In Progress", "Backup Completed", "Restore In Progress", "Completed", "Not Associated"];
@@ -4361,7 +4362,10 @@
                 }
 
                 // Early return ONLY if the Worker AND the Tracked Devices list are both completely unchanged
-                if (state.LastUpdated === globalDataLastUpdated && apiData.length > 0 && currentTrackedHash === globalTrackedHash) {
+                // The worker publishes rows several times a cycle (DataVersion); older workers only
+                // once per cycle (LastUpdated)
+                const dataUnchanged = state.DataVersion ? state.DataVersion === globalDataVersion : state.LastUpdated === globalDataLastUpdated;
+                if (dataUnchanged && apiData.length > 0 && currentTrackedHash === globalTrackedHash) {
                     return; 
                 }
                 
@@ -4374,6 +4378,7 @@
                     const payload = await dataRes.json();
                     if (payload.LastUpdated && payload.Devices) {
                         apiData = payload.Devices;
+                        globalDataVersion = payload.DataVersion || state.DataVersion || "";
                         globalDataLastUpdated = payload.LastUpdated;
                         btn.innerHTML = `<span class="st-pre">Data Updated At: </span>${globalDataLastUpdated}`;
                     } else {

web/layout.css

@@ -333,7 +333,7 @@ table.xf-table thead th:has(> .xf-btn):not(.sortable-th) { position: relative; p
 .xf-bar { display: none; }
 @media (max-width: 1100px) {
     .xf-bar {
-        display: flex; flex-wrap: wrap; gap: 6px; padding: 8px 10px; overflow: visible;   /* chips wrap: no sideways scrolling */
+        display: flex; flex-wrap: wrap; align-items: center; gap: 6px; padding: 8px 10px; overflow: visible;   /* one short row: Filter & sort (+ Card fields / Clear) */
         position: sticky; top: var(--sticky-top, 112px); z-index: 25;
         background: linear-gradient(rgba(0,0,0,0.15), rgba(0,0,0,0.15)), var(--bg-card);
         border-bottom: 1px solid var(--border-color); box-shadow: 0 4px 8px rgba(0,0,0,0.25);
@@ -509,3 +509,29 @@ a.nav-brand:focus-visible { outline: 2px solid #3498db; outline-offset: 3px; }
 
 /* Excel filter: three list modes */
 .xf-seg3 .xf-seg-btn { font-size: 0.78em; padding-left: 4px; padding-right: 4px; }
+
+/* Phones / tablets: the "Filter & sort" button and its column list */
+.xf-chip.xf-filterbtn { font-weight: 600; padding: 7px 13px; }
+.xf-chip.xf-filterbtn svg:first-child { width: 12px; height: 12px; }
+.xf-picker .xf-pick-list { overflow-y: auto; min-height: 0; flex: 1 1 auto; padding: 0 8px 8px; }
+.xf-pick-col {
+    display: flex; align-items: center; gap: 10px; width: 100%; padding: 13px 10px; text-align: left; cursor: pointer;
+    background: none; border: 0; border-bottom: 1px solid var(--border-light); color: var(--text-main); font: inherit;
+}
+.xf-pick-col:hover, .xf-pick-col:focus-visible { background: var(--hover-bg); }
+.xf-pick-name { flex: 1 1 auto; min-width: 0; font-weight: 600; }
+.xf-pick-state { flex: none; font-size: 0.8em; color: var(--text-muted); }
+.xf-pick-col.xf-on .xf-pick-state { color: #b7950b; font-weight: 600; }
+[data-theme="dark"] .xf-pick-col.xf-on .xf-pick-state { color: #f1c40f; }
+.xf-pick-chev { width: 14px; height: 14px; flex: none; color: var(--text-muted); }
+
+/* Card fields: the dialog keeps a fixed height and the two halves share it evenly */
+.xf-fields-overlay .modal-content.xf-fields-sheet { height: calc(100vh - 100px); height: calc(100dvh - 100px); }
+@media (max-width: 1100px) { .xf-fields-overlay .modal-content.xf-fields-sheet { height: calc(100vh - 80px) !important; height: calc(100dvh - 80px) !important; } }
+@media (max-width: 600px) { .xf-fields-overlay .modal-content.xf-fields-sheet { height: calc(100dvh - 72px) !important; } }
+.xf-fields-sheet .xf-fields { flex: 1 1 0; min-height: 120px; max-height: none; }
+.xf-fields-sheet .xf-add { flex: 1 1 0; min-height: 150px; margin-top: 10px; }
+.xf-fields-sheet .xf-add .xf-list { flex: 1 1 auto; min-height: 0; max-height: none; overflow-y: auto; }
+
+/* Details title: when the toggle has to wrap under the name it sits on the left */
+#detailsTitle.dt-wrapped .dt-toggle { margin-left: 0; }

web/layout.js

@@ -59,6 +59,17 @@
         }
         return null;
     }
+    // Details dialog title: if the Simple/Advanced toggle wrapped under the name, align it left
+    function updateDetailsTitle() {
+        const t = document.getElementById('detailsTitle');
+        if (!t || !t.offsetParent) return;
+        const name = t.querySelector('.dt-name'), btn = t.querySelector('.dt-toggle');
+        if (!name || !btn) return;
+        t.classList.remove('dt-wrapped');
+        const wrapped = btn.getBoundingClientRect().top > name.getBoundingClientRect().bottom - 4;
+        t.classList.toggle('dt-wrapped', wrapped);
+    }
+
     function updateSticky() {
         const root = document.documentElement;
         const nav = document.querySelector('.top-nav');
@@ -77,6 +88,7 @@
             if (table.parentElement) setVar(table.parentElement, '--sticky-top', top);   // for the filter chip bar
         });
         setupCustomHScroll();
+        updateDetailsTitle();
     }
 
     // Custom View can be wider than the page. A sideways-scrolling box would stop its header