Kolpack-Software-Consulting/SPRUCE-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source 
Add spruce scraper with CLI, session management, parsers, progress tracking,
recheck logic, and test suite. Includes example config and README.
This commit is contained in:
poprhythm
2026-04-22 10:41:18 -04:00
commit e122f6435a
23 changed files with 3789 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fixtures/scan_view.html
Vendored
+663
View File
@@ -0,0 +1,663 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
<head>
<title>Root View</title>
<!-- <script src="menuscript.js" language="javascript" type="text/javascript"></script> -->
<link rel="stylesheet" type="text/css" href="menustyle.css" media="screen, print" />
<script src="menuscript.js" language="javascript" type="text/javascript"></script>
</head>
<body>
<style>
P {
font: 9pt Arial, Helvetica, sans-serif;
}
.header {
font: bold 22pt Arial, Helvetica, sans-serif;
color: #008080;
text-align: center;
}
.header_desc {
font: bold 10pt Arial, Helvetica, sans-serif;
color: #800000;
text-align: center;
}
.error {
font: bold 16pt Arial, Helvetica, sans-serif;
color: #ff0000;
text-align: center;
}
.information {
font: 10pt Arial, Helvetica, sans-serif;
color: #0000ff;
text-align: left;
}
.data_title {
font: bold 9pt Arial, Helvetica, sans-serif;
color: white;
background: #696969;
text-align: left;
}
.data_odd {
font: 9pt Arial, Helvetica, sans-serif;
font-weight : lighter;
background-color : #eeeeee;
color : #000000;
}
.data_even {
font: 9pt Arial, Helvetica, sans-serif;
font-weight : lighter;
background-color : #ffffff;
color : #000000;
}
.data {
font: 9pt Arial, Helvetica, sans-serif;
color: #000000;
}
.login {
font: bold 9pt Arial, Helvetica, sans-serif;
}
.header_link {
font: bold 9pt Arial, Helvetica, sans-serif;
color: white;
background: #696969;
text-align: left;
text-decoration: none;
}
#pointer_div {
position:relative;
border-style:solid;
border-width:2px;
border-color:red;
margin:5px;
/*cursor:crosshair;*/
}
#data_left {
text-align: left;
}
#image_frame {
background-color: #70a0b0;
font-size:2px;
}
.red_text {
color: #ff0000;
font: bold 9pt Arial, Helvetica, sans-serif;
}
</style>
<script>
function validateFilename(filenameID) {
var item = getObj(filenameID);
var result = item.value;
// based on code found on http://www.codingforums.com/showthread.php?t=194468
if (/^[a-z0-9\.\s\-\_\[\]]*$/i.test(result) === false) { // anything but a-zA-Z0-9, [,], dot, hypen, space, underscore is disallowed
alert ("File name " + result + " contains invalid character(s)! \r\nAnything but a-z, A-Z, 0-9, square brackets, dot, hypen, space, underscore is disallowed");
item.focus();
return false;
}
// If we allow using spaces in filenames, we should at least strip any leading or trailing spaces
item.value = item.value.replace(/^\s+|\s+$/g,""); // the g switch is essential!!
return true;
}
function getObj(ElementId)
{
if (document.getElementById) // BYI now this method works perfect for both IE and NS
return document.getElementById(ElementId);
return false;
}
function showit(ElementId)
{
getObj(ElementId).style.display='block';
}
function hideit(ElementId)
{
getObj(ElementId).style.display='none';
}
function FormatFloat(v)
{
if (v=="")
return v;
return Math.round(v*100)/100;
}
function validateNumber(objId){
// BYI the function validates if the value of the data entry element on the form is an integer or a float
// an alert message is popped up in case of error, the problem field is focused
// the function returns false so that if used in OnSubmit clause it will force the user to re-enter the data
var item = getObj(objId);
var result = item.value;
if (isNaN(Number(result))) {
alert("The value must be an integer or float value.");
item.focus();
return false;
}
return true;
}
function validatePositiveNumber(objId) {
var item = getObj(objId);
var result = item.value;
if (!validateNumber(objId))
return false;
if (result < 0) {
alert("The value must be a positive value.");
item.focus();
return false;
}
return true;
}
</script>
<script>
function PanicStop()
{
// srn - do not ask for confirmation
//if (!confirm("Are you sure you want to stop all process?"))
// return;
var f = document.stop_form;
f.submit();
}
</script>
<form name="stop_form" action="index.php" method="post">
<input type="hidden" name="cmd" value="stop">
</form>
<!-- Button menuing system added 8/2/2009 by gbr -->
<table border="0" cellpadding="0" cellspacing="0" width=100%><tr><td>
<a href="index.php?cmd=scan"
onmouseover="setOverImg('2','');overSub=true;showSubMenu('submenu2','button2');"
onmouseout="setOutImg('2','');overSub=false;setTimeout('hideSubMenu(\'submenu2\')',delay);" target="">
<img src="buttons/button2up.png" border="0" id="button2" vspace="0" hspace="0"></a>
<a href="index.php?cmd=about&mode=photos"
onmouseover="setOverImg('5','');overSub=true;showSubMenu('submenu5','button5');"
onmouseout="setOutImg('5','');overSub=false;setTimeout('hideSubMenu(\'submenu5\')',delay);" target="">
<img src="buttons/button5up.png" border="0" id="button5" vspace="0" hspace="0"></a>
<a href="index.php?cmd=about"
onmouseover="setOverImg('6','');overSub=true;showSubMenu('submenu6','button6');"
onmouseout="setOutImg('6','');overSub=false;setTimeout('hideSubMenu(\'submenu6\')',delay);" target="">
<img src="buttons/button6up.png" border="0" id="button6" vspace="0" hspace="0"></a>
<a href="index.php?cmd=movies"
onmouseover="setOverImg('9','');overSub=true;showSubMenu('submenu9','button9');"
onmouseout="setOutImg('9','');overSub=false;setTimeout('hideSubMenu(\'submenu9\')',delay);" target="">
<img src="buttons/movies_up.png" border="0" id="button9" vspace="0" hspace="0"></a>
<a href="index.php?cmd=logoff"
onmouseover="setOverImg('10','');overSub=true;showSubMenu('submenu10','button10');"
onmouseout="setOutImg('10','');overSub=false;setTimeout('hideSubMenu(\'submenu10\')',delay);" target="">
<img src="buttons/button9up.png" border="0" id="button10" vspace="0" hspace="0"></a><br>
</td></tr></table>
<p class="header">"Plot 20 AMR26 Full Tube Scan " Scan View</p>
<script type="text/javascript" src="js/prototype.js" ></script>
<script type="text/javascript" >
function getcordsInDiv(e){
//get the position of the container
var containerLeft = Position.page($('pointer_div'))[0];
var absLeft = Position.realOffset($('pointer_div'))[0];
var containerTop = Position.page($('pointer_div'))[1];
var absTop = Position.realOffset($('pointer_div'))[1];
//get the mouse coordinates
mouseX = Event.pointerX(e);
mouseY = Event.pointerY(e);
//calculate the absolute mouse position in the div
horizontalPosition = mouseX - containerLeft;
verticalPosition = mouseY - containerTop;
//use prototypes function to get the dimension
//this is a VERY usefull function because it also checks for borders
containerDimensions = $('pointer_div').getDimensions();
height = containerDimensions.height;
width = containerDimensions.width;
//check if the mouse is out or inside the div
// if(horizontalPosition < 0 || verticalPosition < 0 || mouseX > (width + containerLeft) || mouseY > (height + containerTop) ){
if(horizontalPosition < 0 || verticalPosition < 0 ){
TooltipString = 'hp='+horizontalPosition+',vp='+verticalPosition;
}else{
if (!RootDown) {
var xxx = Math.floor((horizontalPosition - absLeft)/33);
var yyy = Math.floor((height - verticalPosition + absTop - 4)/25);
var xx = Math.round(100*((horizontalPosition - absLeft)/33*3.01+0))/100;
var yy = Math.round(100*((height - verticalPosition + absTop - 4)/25*2.26+0))/100;
}
else {
var xxx = Math.floor((width - horizontalPosition + absLeft - 4)/33);
var yyy = Math.floor((verticalPosition - absTop)/25);
var xx = Math.round(100*((width - horizontalPosition + absLeft - 4)/33*3.01+0))/100;
var yy = Math.round(100*((verticalPosition - absTop)/25*2.26+0))/100;
}
TooltipString = 'Tiles X=' + xxx + ', Y=' + yyy + '<br>'+
// 'h=' + height + ', w=' + width + ', tt=' + (absTop+containerTop);
'Xmm=' + xx + ', Ymm=' + yy + '<br>' +
'(Depth undefined offline)';
}
}
Event.observe(document, 'mousemove', getcordsInDiv);
var TooltipString = '';
var tooltip = {
options: {
attr_name: "tooltip",
blank_text: "(????????? ? ????? ????)",
newline_entity: " ",
max_width: 0,
delay: 100,
skip_tags: ["link", "style"]
},
t: document.createElement("DIV"),
c: null,
g: false,
canvas: null,
m: function(e){
if (tooltip.g){
var x = window.event ? event.clientX + (tooltip.canvas.scrollLeft || document.body.scrollLeft) : e.pageX;
var y = window.event ? event.clientY + (tooltip.canvas.scrollTop || document.body.scrollTop) : e.pageY;
tooltip.a(x, y);
}
},
d: function(){
tooltip.canvas = document.getElementsByTagName(document.compatMode && document.compatMode == "CSS1Compat" ? "HTML" : "BODY")[0];
tooltip.t.setAttribute("id", "tooltip");
document.body.appendChild(tooltip.t);
if (tooltip.options.max_width) tooltip.t.style.maxWidth = tooltip.options.max_width + "px"; // all but ie
var a = document.all && !window.opera ? document.all : document.getElementsByTagName("*"); // in opera 9 document.all produces type mismatch error
var l = a.length;
for (var i = 0; i < l; i++){
if (!a[i] || tooltip.options.skip_tags.in_array(a[i].tagName.toLowerCase())) continue;
var tooltip_title = a[i].getAttribute("title"); // returns form object if IE & name="title"; then IE crashes; so...
if (tooltip_title && typeof tooltip_title != "string") tooltip_title = "";
var tooltip_alt = a[i].getAttribute("alt");
var tooltip_blank = a[i].getAttribute("target") && a[i].getAttribute("target") == "_blank" && tooltip.options.blank_text;
if (tooltip_title || tooltip_blank){
a[i].setAttribute(tooltip.options.attr_name, tooltip_blank ? (tooltip_title ? tooltip_title + " " + tooltip.options.blank_text : tooltip.options.blank_text) : tooltip_title);
if (a[i].getAttribute(tooltip.options.attr_name)){
a[i].removeAttribute("title");
if (tooltip_alt && a[i].complete) a[i].removeAttribute("alt");
tooltip.l(a[i], "mouseover", tooltip.s);
tooltip.l(a[i], "mouseout", tooltip.h);
}
}else if (tooltip_alt && a[i].complete){
a[i].setAttribute(tooltip.options.attr_name, tooltip_alt);
if (a[i].getAttribute(tooltip.options.attr_name)){
a[i].removeAttribute("alt");
tooltip.l(a[i], "mouseover", tooltip.s);
tooltip.l(a[i], "mouseout", tooltip.h);
}
}
if (!a[i].getAttribute(tooltip.options.attr_name) && tooltip_blank){
//
}
}
document.onmousemove = tooltip.m;
window.onscroll = tooltip.h;
tooltip.a(-99, -99);
},
_: function(s){
s = s.replace(/\&/g,"&amp;");
s = s.replace(/\</g,"&lt;");
s = s.replace(/\>/g,"&gt;");
return s;
},
s: function(e){
if (typeof tooltip == "undefined") return;
var d = window.event ? window.event.srcElement : e.target;
if (!d.getAttribute(tooltip.options.attr_name)) return;
var s = d.getAttribute(tooltip.options.attr_name);
if (tooltip.options.newline_entity){
var s = tooltip._(s);
s = s.replace(eval("/" + tooltip._(tooltip.options.newline_entity) + "/g"), "<br />");
tooltip.t.innerHTML = s;
}else{
if (tooltip.t.firstChild) tooltip.t.removeChild(tooltip.t.firstChild);
tooltip.t.appendChild(document.createTextNode(s));
}
tooltip.c = setTimeout(function(){
tooltip.t.style.visibility = 'visible';
}, tooltip.options.delay);
tooltip.g = true;
},
h: function(e){
if (typeof tooltip == "undefined") return;
tooltip.t.style.visibility = "hidden";
if (!tooltip.options.newline_entity && tooltip.t.firstChild) tooltip.t.removeChild(tooltip.t.firstChild);
clearTimeout(tooltip.c);
tooltip.g = false;
tooltip.a(-99, -99);
},
l: function(o, e, a){
if (o.addEventListener) o.addEventListener(e, a, false); // was true--Opera 7b workaround!
else if (o.attachEvent) o.attachEvent("on" + e, a);
else return null;
},
a: function(x, y){
var w_width = tooltip.canvas.clientWidth ? tooltip.canvas.clientWidth + (tooltip.canvas.scrollLeft || document.body.scrollLeft) : window.innerWidth + window.pageXOffset;
var w_height = window.innerHeight ? window.innerHeight + window.pageYOffset : tooltip.canvas.clientHeight + (tooltip.canvas.scrollTop || document.body.scrollTop); // should be vice verca since Opera 7 is crazy!
if (document.all && document.all.item && !window.opera) tooltip.t.style.width = "300px"; //tooltip.options.max_width && tooltip.t.offsetWidth > tooltip.options.max_width ? tooltip.options.max_width + "px" : "auto";
var t_width = tooltip.t.offsetWidth;
var t_height = tooltip.t.offsetHeight;
tooltip.t.style.left = x + 15 + "px";
tooltip.t.style.top = y + 8 + "px";
if (x + t_width > w_width) tooltip.t.style.left = -10 + w_width - t_width + "px";
if (y + t_height > w_height) tooltip.t.style.top = -45 + w_height - t_height + "px";
tooltip.t.innerHTML = TooltipString;
tooltip.t.style.width = 210 + "px";
// should be 55 instead of 40 for one more line
tooltip.t.style.height = 55 + "px";
}
}
Array.prototype.in_array = function(value){
var l = this.length;
for (var i = 0; i < l; i++)
if (this[i] === value) return true;
return false;
};
var root = window.addEventListener || window.attachEvent ? window : document.addEventListener ? document : null;
if (root){
if (root.addEventListener) root.addEventListener("load", tooltip.d, false);
else if (root.attachEvent) root.attachEvent("onload", tooltip.d);
}
</script>
<style>
#tooltip{
background:#FFFF00;
border:1px solid #666666;
color:#333333;
font:menu;
margin:0px;
padding:3px 5px;
position:absolute;
visibility:hidden;
width : 180 px;
height : 35 px;
}
</style>
<script type=text/javascript>
var ScaleFactor = 1;
function show_tile(x,y)
{
var TilePopup=window.open("include/tile_view.php?cmd=image&mode=image_scan&id=158374&sX=0&sY=0&eX=310&eY=740&dX=3.01&dY=2.26&rd=1&sz=1"+"&s="+ScaleFactor+"&x="+x+"&y="+y+"&Tx="+Tx+"&Ty="+Ty, 'RTL_window',"width=670,height=540,left=200,top=150, titlebar=no,location=no,toolbar=no");
TilePopup.focus();
}
</script>
<script type="text/javascript"> <!--
var tt, ll, bb, rr = "";
var RootDown = 1;
running = 1;
var t, l, b, r = "";
var Tx = -1,
Ty = -1;
function clearTile() {
if ($('my_pointer') )
$('pointer_div').removeChild($('my_pointer'));
}
function hiliteTile() {
var xTile, yTile;
var rootGrowsDown;
xTile = $('xTile').value;
yTile = $('yTile').value;
rootGrowsDown = 1;
if (validatePositiveNumber('xTile')) {
xx = 102;
if (xTile > xx) {
alert("The max value for X tile is "+ xx+" (we count from zero tile)");
$('xTile').focus();
return false;
}
}
else
return false;
if (validatePositiveNumber('yTile')) {
yy = 327;
if (yTile > yy) {
alert("The max value for Y tile is "+ yy+" (we count from zero tile)");
$('yTile').focus();
return false;
}
}
else
return false;
clearTile();
my_pointer = document.createElement("div");
my_pointer.setAttribute("id","my_pointer");
if (rootGrowsDown)
my_pointer.setAttribute("style", "background-color:#6ff;filter:alpha(opacity=75);opacity:0.75;width:17px;height:9px;border:3px solid #f00;position:absolute;top:" + (25*yTile + 5) + "px;left:" + (33*(102- xTile)+5) + "px;");
else
my_pointer.setAttribute("style", "background-color:#6ff;filter:alpha(opacity=75);opacity:0.75;width:17px;height:9px;border:3px solid #f00;position:absolute;top:"+(25*(327- yTile)+5)+"px;left:"+(33*xTile+5)+"px;");
$('pointer_div').appendChild(my_pointer);
return false;
}
function point_it(event){
pos_x = event.offsetX ? (event.offsetX) : event.pageX - document.getElementById("pointer_div").offsetLeft;
pos_y = event.offsetY?(event.offsetY):event.pageY - document.getElementById("pointer_div").offsetTop;
ll = pos_x;
tt = pos_y;
rr = pos_x;
bb = pos_y;
fx(pos_y,pos_x,0);
}
function fx(top,left, FirstTime){
if (running == 2){
current = document.getElementById('pointer');
current.parentNode.removeChild(current);
running = 1;
}
if (running == 1){
t=top;
l=left;
b=top;
r=left;
// now round up the rectangle to the covering tiles
Tx = Math.floor(l/33);
Ty = Math.floor(t/25);
t = 25* Ty;
l = 33* Tx;
b = 25* Math.ceil(b/25);
r = 33* Math.ceil(r/33);
if (RootDown == 0) {
Ty = 328 - Ty - 1;
rectL = Math.floor(ll/33)*3.01+0;
rectB = 2.26*(328 - Math.floor(bb/25) -1)+0;
}
else {
rectL = 3.01*(103 - Math.floor(ll/33) -1)+0;
rectB = Math.floor(bb/25)*2.26+0;
}
// Tx = Math.round(rectL/3.01);
// Ty = Math.round(rectB/2.26);
element = document.createElement("div");
element.setAttribute("id","pointer");
// element.setAttribute("style", "background-color:#fff;filter:alpha(opacity=75);opacity:0.75;width:"+(r-l)+"px;height:"+(b-t)+"px;border:1px solid #f00;position:relative;top:"+(t-8200-7)+"px;left:"+(l-2)+"px;");
element.setAttribute("style", "background-color:#fff;filter:alpha(opacity=75);opacity:0.75;width:"+(r-l-22)+"px;height:"+(b-t-18)+"px;border:4px solid #0F0;position:relative;top:"+(t-8200)+"px;left:"+(l+6)+"px;");
document.getElementById('pointer_div').appendChild(element);
running=2;
if (!FirstTime)
show_tile(rectL,rectB);
}
}
function showSubscanPage() {
subscanURL = 'index.php?cmd=scan&mode=subscan&id=158374';
if (Tx != -1)
subscanURL = subscanURL + "&Tx=" + Tx + "&Ty=" + Ty;
window.location.href=subscanURL;
}
//-->
</script>
<center>
<table id="data_left" cellpadding=0 cellspacing=3 border=0 width="100%">
<tr><td valign="top" width="35%">
<table cellpadding=0 cellspacing=3 border=0>
<tr><td>Scan ID:</td><td>158374</td></tr>
<tr><td>Name:</td><td>Plot 20 AMR26 Full Tube Scan </td></tr>
<tr><td>Scan Time:</td><td>2024-07-29 05:00:00</td></tr>
<tr><td>Starting X:</td><td>0 mm</td></tr>
<tr><td>Starting Y:</td><td>0 mm</td></tr>
<tr><td>Ending X:</td><td>310 mm</td></tr>
<tr><td>Ending Y:</td><td>740 mm</td></tr>
<tr><td>DX:</td><td>3.01 mm</td></tr>
<tr><td>DY:</td><td>2.26 mm</td></tr>
<tr><td>Dwell Time:</td><td>100 ms</td></tr>
<tr><td>Scan Lines:</td><td>Horizontal</td></tr>
<tr><td>Scan Mode:</td><td>Raster</td></tr>
<tr><td>Step Units:</td><td>mm</td></tr>
<tr><td>Start Time:</td><td>2024-07-29 04:59:46</td></tr>
<tr><td>End Time:</td><td>2024-07-30 02:51:07</td></tr>
<tr><td>Scan Status:&nbsp;&nbsp;&nbsp;</td><td>Completed</td></tr>
<tr><td>Root grows down:&nbsp;&nbsp;</td><td>Yes</td></tr>
<tr><td>Notes:</td><td></td></tr>
<tr><td>User:</td><td>SPRUCE</td></tr>
<tr><td>Total number of images:</td><td>33784 (103x328)</td>
</tr>
<tr><td>Total Disk Space:</td><td>1949.001 Mb</td></tr>
<tr><td>Total Travel distance:</td><td>204098.72 mm</td></tr>
<tr><td nowrap>Estimated Scan Time (HH:MM:SS):</td><td>
15:05:16 </td>
<tr>
<td>Scan Time (HH:MM:SS):</td>
<td>21:51:21</td>
</tr>
</table>
</td>
<td valign="top" width="65%">
<form method="post" name="imageform" action="index.php">
<input type="hidden" name="cmd" value="scan">
<input type="hidden" name="mode" value="view">
<input type="hidden" name="id" value="158374">
<input type="hidden" name="fZ" value="1">
<table cellpadding=0 cellspacing=0 border=0>
<tr>
<td>&nbsp;</td>
<td>Popup NxN:&nbsp;&nbsp;</td>
<td><select name="fS" onclick="ScaleFactor=this.value;">
<option value="1" selected>1X1</option>
<option value="2">2X2</option>
<option value="3">3X3</option>
<option value="4">4X4</option>
<option value="5">5X5</option>
<option value="6">6X6</option>
<option value="7">7X7</option>
</select></td>
<td>&nbsp;&nbsp;</td>
<td>&nbsp;
</td>
<td>&nbsp;</td>
<td>
<!-- this works just fine as a traditional submit button: <input type="button" value="Switch to Subscan Mode" onClick="window.location.href='index.php?cmd=scan&mode=subscan&id=158374'"> -->
<!-- however, this will use an image as a button, and does the same thing as the line above. gbr 10/3/2009 -->
<input type=image src="buttons\tosubscan.png" onClick="showSubscanPage(); return false;">
&nbsp;&nbsp;</td>
<td>&nbsp;</td>
</tr>
<tr>
<td>&nbsp;</td>
<td colspan=3>Highlight tile at X:&nbsp;
<input type="text" name="xTile" id="xTile" value="0" size="4" id="xTile">,&nbsp; Y:&nbsp;<input type="text" name="yTile" id="yTile" value="0" size="4" id="yTile"> &nbsp;
</td><td colspan=3><input type=image src="buttons\do_it_up.png" ID=toggler ONCLICK="hiliteTile(); return false;">&nbsp;<input type=image src="buttons\clear_up.png" ID=toggler1 ONCLICK="clearTile(); return false;"></td><td>&nbsp;</td>
</tr>
</table>
</form>
<div id="pointer_div" onclick="point_it(event)" style = "position:absolute;width:3399px;height:8200px;">
<img id ="container" title="Please wait while the mosaic is loaded completely..." src="http://205.149.147.131:8011/RootView_Database/158374/mosaic.jpg">
</div>
</td>
</tr>
</table>
<script type="text/javascript">
if (Tx != -1) {
if (1 == 0) {
ll = rr = Tx * 33 + 1;
tt = bb = (328 - Ty - 1) * 25 + 1;
}
else {
ll = rr = (103 - Tx -1) * 33 + 1;
tt = bb = Ty * 25 + 1;
}
fx(tt, ll, 1);
}
</script>