diff --git a/CONTROL/control b/CONTROL/control index 3644b803..422eca4b 100644 --- a/CONTROL/control +++ b/CONTROL/control @@ -1,7 +1,7 @@ Package: webif Priority: optional Section: web -Version: 1.3.4-14 +Version: 1.3.5 Architecture: mipsel Maintainer: af123@hpkg.tv Depends: tcpfix,webif-channelicons(>=1.1.24),lighttpd(>=1.4.39-1),jim(>=0.77),jim-oo(>=0.77),jim-sqlite3(>=0.76),jim-cgi(>=0.7-1),jim-binary(>=0.76),service-control(>=2.3),busybox(>=1.20.2-1),lsof(>=4.87),epg(>=1.2.5),hmt(>=2.0.10),ssmtp,cron-daemon(>=1.18.3-3),at(>=3.1.18),anacron,trm(>=1.1),openssl-command,nicesplice,id3v2,file,rsvsync(>=1.1.9),webif-charts(>=1.2-1),stripts(>=1.2.5-3),tmenu(>=1.08),ffmpeg,id3v2,multienv(>=1.6),tcpping(>=1.1),e2fsprogs,wireless-tools(>=29-1),dbupdate,recmon(>=2.0.7),hwctl,nugget(>=0.95),sqlite3(>=3.15.1) diff --git a/etc/recmon.d/autotrigger b/etc/recmon.d/autotrigger index d649e25c..ad1102ba 100755 --- a/etc/recmon.d/autotrigger +++ b/etc/recmon.d/autotrigger @@ -17,14 +17,14 @@ if {$loglevel > 0} { } # Wait up to 10 minutes for any existing auto process to finish... -if {![acquire_lock webif_auto 600 5]} { +if {![acquire_lock webif_autoscan 600 5]} { puts "Cannot acquire exclusive lock, terminating." system plog auto "$prefix: failed to get lock" exit } if {$loglevel > 0} { system plog auto "$prefix: got lock" } -exec /mod/webif/lib/bin/auto -prelocked -logprefix "$prefix: " -singledir $dir +exec /mod/webif/lib/auto/scan -prelocked -logprefix "$prefix: " -singledir $dir if {$loglevel > 0} { system plog auto "$prefix: done" } diff --git a/webif/cgi-bin/status.jim b/webif/cgi-bin/status.jim index 18cce480..79cb26ec 100755 --- a/webif/cgi-bin/status.jim +++ b/webif/cgi-bin/status.jim @@ -39,7 +39,7 @@ set statusops { mpg { "Extracting MPG" "/img/mpg.png style=\"padding: 0 0.2em 0 0.5em\"" } - mp3 { "Extracting MP3" + mp { "Extracting MP3" "/img/mp3.png style=\"padding: 0 0.2em 0 0.5em\"" } shrink { "Shrinking" diff --git a/webif/html/diag/queue/fetch.jim b/webif/html/diag/queue/fetch.jim index 733c0d55..2e557da4 100755 --- a/webif/html/diag/queue/fetch.jim +++ b/webif/html/diag/queue/fetch.jim @@ -14,23 +14,24 @@ foreach q [queue all] { "/media/" "" ".ts" "" } [$q get file]] - set dat [clock format [$q get dat] -format {%c}] + set submitted [clock format [$q get submitted] -format {%c}] if {$flag} { puts "," } else { incr flag } puts "{" puts " \"qid\": [$q get id]," - puts " \"dat\": \"$dat\"," + puts " \"submitted\": \"$submitted\"," puts " \"file\": \"[::json::escape $name]\"," puts " \"action\": \"[$q get action]\"," puts " \"status\": \"[$q get status]\"," puts " \"log\": \"[::json::escape [$q get log]]\"," - if {[$q get elapsed] > 0} { - set time [clock format [$q get elapsed] -format "%T"] - puts " \"elapsed\": \"$time\"," + if {[$q get runtime] > 0} { + set time [clock format [lindex [split [$q get runtime] .] 0] \ + -format "%T"] + puts " \"runtime\": \"$time\"," } else { - puts " \"elapsed\": \"0\"," + puts " \"runtime\": \"0\"," } - puts " \"retry\": \"[$q get retry]\"" + puts " \"retries\": \"[$q get retries]\"" puts -nonewline "}" } puts "\n]" diff --git a/webif/html/diag/queue/script.js b/webif/html/diag/queue/script.js index 9ad594b7..1c5d0dd1 100644 --- a/webif/html/diag/queue/script.js +++ b/webif/html/diag/queue/script.js @@ -20,15 +20,15 @@ function load() '' + v.qid + '' + - '' + v.dat + '' + + '' + v.submitted + '' + '' + v.file + '' + '' + v.action + '' + '' + v.status; if (v.status == 'RUNNING') s += '  '; s += ''; - if (v.elapsed != '0') - s += v.elapsed; + if (v.runtime != '0') + s += v.runtime; s += '' + '' + v.log + '' + ''; diff --git a/webif/lib/auto/NOTES b/webif/lib/auto/NOTES new file mode 100644 index 00000000..45a7de6c --- /dev/null +++ b/webif/lib/auto/NOTES @@ -0,0 +1,148 @@ + +Everything is a plugin. Those bundled with the web interface are in +/mod/webif/lib/auto/ and others are in /mod/webif/plugin/ + +---------------------------------------------------------------------- +Priorities (* - additional package) +---------- + +Module Scan De-queue +------ ---- -------- +*newk 1000 - + expire 900 - + dedup 800 - +*sweeper 700 - +*flatten 650 - +*flatview 650 - + decrypt 600 900 + shrink 400 800 + mpg 300 300 + mp3 300 300 +*detectads 300 200 + +Notes: + +* Anything that moves files around runs first. + (used to always prefer decryption first to improve the changes that the + file was DLNA indexed but DLNA helper removes that constraint) +* Decrypt > Shrink so arbookmarks can run post-decrypt and still have the + EIT tables present. + +---------------------------------------------------------------------- +Scan phase hooks +---------------- + +The scan process looks for 'auto.hook' + +Plugins register themselves with: + + ::auto::register [priority] + + If not specified, priority defaults to 500. + +The framework will call the following callbacks (if defined within the module): + + ::module::init + + This function will be called before any module runs and can be + used if the module requires any initialisation. + + ::module::cleanup + + Called once all modules have completed their work. + + ::module::run + + This function is called when the module should do its work against + all recordings. + + ::module::rundir + + The module should do its work against the specific provided directory. + +The following global variables are available for modules to use: + + ::auto::settings - instance of the system settings class. + ::auto::root - root directory of recordings. + ::auto::dustbin - path to the dustbin directory. + +The following functions are available for modules to use: + + ::auto::log [level] + + Log a message to the log file with appropriate prefix. + The level argument should be a number between 0 and 2 inclusive. + If level is not provided then it defaults to 1. + Level 2 is reserved for debugging messages. + + ::auto::dsc [required bytes] + + Checks that there is enough available system disk space + (1GiB + 3 * ) and aborts processing otherwise. + If [required bytes] is not provided then it defaults to zero and + the function just checks fro 1GiB of free disk space. + + ::auto::inuse + + Check if a recording is in use (with retries). + + ::auto::autoflagscan + + Starting at the directory indicated by , all directories are + recursively scanned looking for those containing a file named + ".auto" or ".autor". + + This function automatically takes care of avoiding the dustbin, + filesystem loops, disabling recursion when a special folder is found, + etc. It is used by built-in modules such as "decrypt" + + The callback will be called once for each flagged directory with + the directory name passed as the sole argument. + + ::auto::flagscan + + Starting at the directory indicated by , all directories are + recursively scanned looking for those containing a file named + ".". + + The callback will be called once for each flagged directory with + the directory name passed as the sole argument. + + ::auto::direntries + + Scan the named directory and call the callback for each loadable + ts file found. The ts object is passed to the callback function. + + ::auto::recalcdir + + Indicate that the unwatched recording count for the given directory + should be updated at the end of the run. + +---------------------------------------------------------------------- +De-queue hooks +-------------- + + ::auto::register [priority] + + If not specified, priority defaults to 500. + +The framework will call the following callbacks (if defined within the module): + + ::module::dequeue + + De-queue the provided queue item and associated ts file object. + The module must return a list containing [message] + + Possible results: + + OK - The queue item was successfully processed. + FAILED - Processing failed (and should not be re-attempted). + DEFER - Processing should be tried again later. + + Examples: + + return "OK" + return [list "OK" "Processing was successful."] + return [list "DEFER" "File not yet decrypted."] + return [list "FAILED" "File is already decrypted."] + diff --git a/webif/lib/auto/decrypt/auto.hook b/webif/lib/auto/decrypt/auto.hook new file mode 100755 index 00000000..848bc537 --- /dev/null +++ b/webif/lib/auto/decrypt/auto.hook @@ -0,0 +1,58 @@ + +proc ::decrypt::ts {ts} { + set file [$ts get file] + + if {![$ts flag "ODEncrypted"]} { + ::auto::log " $file - Already decrypted." 2 + return + } + + if {[$ts flag "Encrypted"]} { + ::auto::log " $file - Protected (Enc flag)." 2 + return + } + + if {[::auto::inuse $ts]} { + ::auto::log " $file - In use." + return + } + + if {[queue check $ts decrypt]} { + ::auto::log " $file - Already queued." 2 + return + } + + # Check that the file is not already decrypted by analysing it. + set anencd [exec /mod/bin/stripts -qE [file rootname $file]] + if {$anencd != "1"} { + ::auto::log " $file - already decrypted but the HMT flag is wrong." 0 + # Fix... + exec /mod/webif/lib/bin/fixencflags $file + return + } + + # Enqueue file + queue insert $ts decrypt + ::auto::log " $file - Queued for decryption." 0 +} + +proc ::decrypt::directory {dir} { + ::auto::log "DECRYPT: \[$dir]" 2 + ::auto::direntries $dir ::decrypt::ts +} + +proc ::decrypt::rundir {dir} { + set sup [::auto::autoflagscanup $dir decrypt] + if {$sup == -1} { + log "Encountered special directory." 2 + return + } + ::auto::autoflagscan $dir decrypt ::decrypt::directory 0 $sup +} + +proc ::decrypt::run {} { + ::auto::autoflagscan $::auto::root decrypt ::decrypt::directory +} + +::auto::register decrypt 600 + diff --git a/webif/lib/auto/decrypt/queue.hook b/webif/lib/auto/decrypt/queue.hook new file mode 100644 index 00000000..23a828ab --- /dev/null +++ b/webif/lib/auto/decrypt/queue.hook @@ -0,0 +1,132 @@ + +proc ::decrypt::dequeue {q ts} { + namespace import ::auto::log + + set tmp $::auto::tmp + set file [$ts get file] + set rfile [file rootname $file] + set bfile [file tail $file] + + if {![$ts flag "ODEncrypted"]} { + return {"OK" "Already decrypted"} + } + + if {[$ts flag "Encrypted"]} { + return {"DEFER" "Protected (Enc flag)"} + } + + if {![system dlnastatus]} { + return {"DEFER" "DLNA Server not running"} + } + + if {[::auto::inuse $ts]} { + return {"DEFER" "Recording in use"} + } + + # Check that the file is not already decrypted by analysing it. + set anencd [exec /mod/bin/stripts -qE $rfile] + if {$anencd != "1"} { + exec /mod/webif/lib/bin/fixencflags $file + return {"OK" + "Already decrypted but the HMT flag was wrong (fixed)"} + } + + lassign [$ts dlnaloc "127.0.0.1"] url + if {$url ne ""} { + log " $file - has been indexed." + set helper 0 + } else { + log " $file - Not yet indexed, trying helper." + if {[catch { + lassign [system dlnahelper [file normalize $file]] url + } msg]} { + log " $file - $msg" + return [list "FAILED" $msg] + } + if {$url eq ""} { + return {"DEFER" "DLNA helper failed"} + } + set helper 1 + } + + # Perform the decryption by requesting the file from the DLNA server. + set size [$ts size] + ::auto::dsc $size + system startop decrypt $file + ::auto::startclock + log " DECRYPT: $rfile" 0 + log " DLNA: $url" 0 + exec wget -O "$tmp/$bfile" $url + + # Release the helper lock once finished. + if {$helper} { system dlnahelper -release } + + if {[file size $file] != [file size "$tmp/$bfile"]} { + log " $file - File size mismatch." 0 + file tdelete "$tmp/$bfile" + system endop decrypt + return {"DEFER" "File size mismatch"} + } + + # Check if the file is in use. It is possible that the file is + # now being played even though it was free when decryption started. + if {[::auto::inuse $ts]} { + log " $file - In use." + file tdelete "$tmp/$bfile" + system endop decrypt + return {"DEFER" "Recording in use"} + } + + # Copy the HMT file over for stripts + set thmt "$tmp/[file rootname $bfile].hmt" + file copy "$rfile.hmt" $thmt + # Check that the file is no longer encrypted by analysing it. + set anencd [exec /mod/bin/stripts -qE "$tmp/[file rootname $bfile]"] + file delete $thmt + + if {$anencd != "0"} { + log " $file - File did not decrypt properly." 0 + file tdelete "$tmp/$bfile" + if {[$q get retries] > 3} { + system notify "$file - auto-decrypt failed." + } + system endop decrypt + return {"DEFER" "Recording did not decrypt properly"} + } + + # Move the encrypted file out of the way. + file rename $file "$rfile.encrypted" + # Move the decrypted copy into place. + file rename "$tmp/$bfile" $file + # Set the file time to match the old file + file touch $file "$rfile.encrypted" + # Patch the HMT - quickest way to get back to a playable file. + exec /mod/bin/hmt -encrypted "$rfile.hmt" + + log " Removing/binning old copy." 0 + # Move the old recording to the bin if undelete is installed. + if {$::auto::dustbin ne ""} { + set bin [_del_bindir $file "webif_autodecrypt"] + set tail [file tail $rfile] + file rename "$rfile.encrypted" "$bin/$tail.ts" + foreach ext {nts hmt thm} { + if {[file exists "$rfile.$ext"]} { + file copy $rfile.$ext "$bin/$tail.$ext" + if {$ext eq "hmt"} { + # Patch the binned HMT back + exec /mod/bin/hmt +encrypted \ + "$bin/$tail.hmt" + } + } + } + } else { + file tdelete "$rfile.encrypted" + } + set summary [::auto::endclock $size] + $ts unflag "ODEncrypted" + system endop decrypt + return [list "OK" $summary] +} + +::auto::register decrypt 900 + diff --git a/webif/lib/auto/dedup/auto.hook b/webif/lib/auto/dedup/auto.hook new file mode 100755 index 00000000..3d6f74f6 --- /dev/null +++ b/webif/lib/auto/dedup/auto.hook @@ -0,0 +1,29 @@ + +proc ::dedup::directory {dir} { + namespace import ::auto::log + + log "DEDUP: \[$dir]" 2 + loop i 0 1 { + foreach line [split \ + [exec /mod/webif/html/dedup/dedup -yes -auto $dir] "\n"] { + log $line 2 + } + } + ::auto::recalcdir $dir +} + +proc ::dedup::rundir {dir} { + set sup [::auto::autoflagscanup $dir dedup] + if {$sup == -1} { + log "Encountered special directory." 2 + return + } + ::auto::autoflagscan $dir dedup ::dedup::directory 0 $sup +} + +proc ::dedup::run {} { + ::auto::autoflagscan $::auto::root dedup ::dedup::directory +} + +::auto::register dedup 800 + diff --git a/webif/lib/auto/deq b/webif/lib/auto/deq new file mode 100755 index 00000000..c97bb7af --- /dev/null +++ b/webif/lib/auto/deq @@ -0,0 +1,205 @@ +#!/mod/bin/jimsh + +source /mod/webif/lib/setup +require system.class settings.class ts.class rsv.class browse.class \ + queue.class \ + lock plugin safe_delete pretty_size + +source /mod/webif/lib/auto/util.jim + +set ::auto::settings [settings] +set ::auto::loglevel [$::auto::settings autolog] + +set ::auto::force 0 + +while {[llength $argv]} { + switch -- [lindex $argv 0] { + -d { + set ::auto::loglevel 2 + set ::auto::logfd stdout + } + -f { + set ::auto::force 1 + } + -logprefix { + set argv [lrange $argv 1 end] + if {[llength $argv]} { + set ::auto::logprefix [lindex $argv 0] + } + } + default { + # Pass to rest of script. + break + } + } + set argv [lrange $argv 1 end] +} + +# Acquire lock +if {$::auto::logfd ne "unset"} { + puts $::auto::logfd "Acquiring lock..." +} +if {![acquire_lock webif_autodeq]} { + if {$::auto::loglevel > 1} { + system plog auto "Could not acquire lock." + } + puts "Could not acquire exclusive lock, terminating." + exit +} + +::auto::loginit + +###################################################################### +# Determine if it's time to run + +if {[system uptime] < 180} { + ::auto::log "Aborting, system has just booted." 2 + exit +} + +::auto::dsc +::auto::oktorun + +######################################################################### +# Initialisation + +set scanstart [clock milliseconds] +::auto::log "Auto de-queue starting" + +::auto::tmpdir "webif_autoq" + +if {[system pkginst undelete]} { + set ::auto::dustbin "[system dustbin]" +} else { + set ::auto::dustbin "" +} + +set ::auto::root [system mediaroot] +file stat "$::auto::root/" rootstat +set ::auto::rootdev $rootstat(dev) + +######################################################################### +# Utility functions + +###################################################################### +# Plugin registration + +set ::auto::plugins {} + +proc ::auto::register {plugin {priority 500}} { + variable plugins + + set fn "::${plugin}::dequeue" + set plugins($plugin) $priority + log "Registered $plugin with priority $priority" +} + +###################################################################### +# Load plugins + +# Bundled +eval_plugins queue 1 "" /mod/webif/lib/auto + +# Third party + +eval_plugins queue 1 + +###################################################################### +# Process the queue + +queue startup [$::auto::settings autokeep] + +proc ::auto::dumpq {qq} { + foreach q $qq { + if {[$q get action] in $::auto::plugins} { + set pri $::auto::plugins([$q get action]) + } + log [format " C: %4d %5d %8s - [$q get file]" \ + [$q get id] $pri [$q get action]] 2 + } +} + +proc ::auto::runplugin {plugin fn args} { + set rfn "::${plugin}::${fn}" + if {![exists -proc $rfn]} { return -1 } + if {[catch {set ret [uplevel 1 $rfn {*}$args]} msg]} { + log "$rfn: $msg" 0 + return -1 + } + return $ret +} + +proc ::auto::runplugins {fn args} { + foreach plugin [dict keys $::auto::plugins] { + set rfn "::${plugin}::${fn}" + if {![exists -proc $rfn]} continue + if {[catch {uplevel 1 $rfn {*}$args} msg]} { + log "$rfn: $msg" 0 + } + } +} + +# Helper function to sort a list of queue items by plugin priority +proc ::auto::pending {} { + return [lsort -decreasing -command [lambda {a b} { + set aa [$a get action] + set ba [$b get action] + if {$aa ni $::auto::plugins} { return 0 } + if {$ba ni $::auto::plugins} { return 0 } + return $($::auto::plugins($aa) - $::auto::plugins($ba)) + }] [queue pending]] +} + +for {set qq [::auto::pending]} {[llength $qq]} {set qq [::auto::pending]} { + ::auto::dumpq $qq + + # Try to run the first item in the queue. + set q [lindex $qq 0] + set plugin [$q get action] + + ::auto::log "De-queuing [$q get id] - [$q get action] - [$q get file]" 0 + + if {[catch {set ts [ts fetch [$q get file]]}] || $ts eq "0"} { + ::auto::log "ts load failed." 0 + $q update "FAILED" "Could not load .ts file" 1 + continue + } + + ::auto::dsc + ::auto::oktorun + + $q update RUNNING "Started at [clock format [clock seconds]]" + + set ologprefix $::auto::logprefix + set ::auto::logprefix "$plugin:$::auto::logprefix" + + set st [clock milliseconds] + lassign [::auto::runplugin $plugin dequeue $q $ts] code msg + set ::auto::logprefix $ologprefix + set elapsed [::auto::elapsed $st] + + ::auto::log " $code - $msg" 0 + switch -- $code { + "-1" { + ::auto::log " Plugin failure." 0 + $q update "FAILED" "Plugin failure" 1 + } + "OK" { + $q update "COMPLETE" $msg 1 $elapsed + ::auto::runplugins dequeued $plugin $q $ts + } + "DEFER" { $q update "DEFER" $msg 1 $elapsed } + "FAILED" { $q update "FAILED" $msg 1 $elapsed } + default { + $q update "FAILED" "Unknown response '$code' from plugin" + } + } +} + +###################################################################### +# Cleanup + +release_lock webif_autodeq + +::auto::log "Auto de-queue completed in [::auto::elapsed $scanstart] seconds." + diff --git a/webif/lib/auto/expire/auto.hook b/webif/lib/auto/expire/auto.hook new file mode 100755 index 00000000..025d77b6 --- /dev/null +++ b/webif/lib/auto/expire/auto.hook @@ -0,0 +1,90 @@ + +proc ::expire::directory {dir} { + namespace import ::auto::log + + log "EXPIRE: \[$dir]" 2 + + # type 0 keep {} days 2 keepnew 0 + set ax [dir expiry $dir] + log " $ax" 2 + if {![string is integer -strict $ax(keep)]} { set ax(keep) 0 } + if {![string is integer -strict $ax(days)]} { set ax(days) 0 } + + if {!$ax(keep) && !$ax(days)} { + log "Expire error ($dir): neither keep nor days set ($ax)" 0 + return + } + + if {$ax(type) eq "0"} { + set getter [lambda {ts} { list [$ts get end] }] + } else { + set getter [lambda {ts} { list [$ts lastmod] }] + } + + set entries [lsort \ + -command [lambda {a b} {upvar getter g; expr [$g $a] - [$g $b]}]\ + [lmap i [readdir -nocomplain $dir] { + if {![string match {*.ts} $i]} continue + if {[catch {set ts [ts fetch "$dir/$i"]}]} continue + if {$ts == 0} continue + list $ts + }]] + + # Entries are now sorted with oldest first, according to the selected + # criterion. + + set num [llength $entries] + log " Entries: $num" 2 + foreach e $entries { log " [file tail [$e get file]]" 2 } + + set now [clock seconds] + foreach ts $entries { + set file [$ts get file] + if {$num <= $ax(keep)} { + log " Remaining ($num) <= keep ($ax(keep))" 2 + break + } + log " [file tail $file]" + if {[::auto::inuse $ts]} { + log " In use, skipping." + continue + } + if {$ax(keepnew) ne "0" && [$ts flag New]} { + log " Unwatched, skipping." + continue + } + if {$ax(days) > 0} { + # Calculate age in days + set age $(($now - [$getter $ts]) / 86400.0) + log " $age days (cf. $ax(days))" + if {$age < $ax(days)} { + log " Too new, skipping." + continue + } + } + log " Removing." + if {![safe_delete $file]} { + log "Unknown error in safe_delete, aborting." 0 + break + } + log " EXPIRE: removed $file" 0 + ::auto::recalcdir $dir + incr num -1 + } +} + +proc ::expire::rundir {dir} { + set sup [::auto::autoflagscanup $dir expire] + if {$sup == -1} { + log "Encountered special directory." 2 + return + } + ::auto::autoflagscan $dir expire ::expire::directory 0 $sup +} + +proc ::expire::run {} { + ::auto::autoflagscan $::auto::root expire ::expire::directory +} + +::auto::register expire 900 + diff --git a/webif/lib/auto/mp3/auto.hook b/webif/lib/auto/mp3/auto.hook new file mode 100755 index 00000000..4482a09b --- /dev/null +++ b/webif/lib/auto/mp3/auto.hook @@ -0,0 +1,45 @@ + +proc ::mp3::ts {ts} { + set file [file rootname [$ts get file]] + + if {[file exists $file.mp3]} { + # Already done. + return + } + + if {[$ts get definition] eq "HD"} { + # Cannot extract a useful MP3 from a HD recording. + ::auto::log " $file - High definition." 2 + return + } + + if {[queue check $ts mp3]} { + ::auto::log " $file - Already queued." 2 + return + } + + # Enqueue file + queue insert $ts mp3 + ::auto::log " $file - Queued for mp3." 0 +} + +proc ::mp3::directory {dir} { + ::auto::log "MP3: \[$dir]" 2 + ::auto::direntries $dir ::mp3::ts +} + +proc ::mp3::rundir {dir} { + set sup [::auto::autoflagscanup $dir mp3] + if {$sup == -1} { + log "Encountered special directory." 2 + return + } + ::auto::autoflagscan $dir mp3 ::mp3::directory 0 $sup +} + +proc ::mp3::run {} { + ::auto::autoflagscan $::auto::root mp3 ::mp3::directory +} + +::auto::register mp3 300 + diff --git a/webif/lib/auto/mp3/queue.hook b/webif/lib/auto/mp3/queue.hook new file mode 100644 index 00000000..40f013c6 --- /dev/null +++ b/webif/lib/auto/mp3/queue.hook @@ -0,0 +1,70 @@ + +set ::mp3::audiomp3 [$::auto::settings audiomp3] + +proc ::mp3::dequeue {q ts} { + namespace import ::auto::log + variable audiomp3 + + set tmp $::auto::tmp + + set file [file rootname [$ts get file]] + + if {[file exists $file.mp3]} { + return {"OK" "Already done"} + } + + if {[$ts flag "ODEncrypted"]} { + return {"DEFER" "Not decrypted"} + } + + if {[$ts get definition] eq "HD"} { + # Cannot extract a useful MP3 from a HD recording. + return {"FAILED" "Cannot process high-definition recording"} + } + + if {[::auto::inuse $ts]} { + return {"DEFER" "Recording in use"} + } + + system startop mp3 [$ts get file] + ::auto::dsc [$ts size] + + ::auto::startclock + + log " MP3: $file" 0 + log " Converting... [$::auto::settings \ + audiomp3descr $audiomp3]" 0 + if {[catch { + set cmd [list nice -n 19 \ + /mod/bin/ffmpeg -y -benchmark -v 0 -i "$file.ts" \ + -f mp3 -vn] + if {!$audiomp3} { lappend cmd -acodec copy } + lappend cmd "$tmp/mp3.mp3" + foreach line [split [exec {*}$cmd] "\n"] { + log [string trim $line] 0 + } + } msg]} { + system notify "$file - auto-mp3 - error $msg." + system endop mp3 + return [list "FAILED" $msg] + } + + if {[system pkginst id3v2]} { + log [exec /mod/bin/id3v2 \ + --song "[$ts get title]" \ + --comment "[$ts get synopsis]" \ + --album "[$ts get channel_name]" \ + --year "[clock format [$ts get start] -format {%Y}]" \ + "$tmp/mp3.mp3"] 0 + } + + # Move the MP3 into the local directory + file rename $tmp/mp3.mp3 $file.mp3 + set summary [::auto::endclock [$ts size]] + system endop mp3 + + return [list "OK" $summary] +} + +::auto::register mp3 300 + diff --git a/webif/lib/auto/mpg/auto.hook b/webif/lib/auto/mpg/auto.hook new file mode 100755 index 00000000..da5930a3 --- /dev/null +++ b/webif/lib/auto/mpg/auto.hook @@ -0,0 +1,45 @@ + +proc ::mpg::ts {ts} { + set file [file rootname [$ts get file]] + + if {[file exists $file.mpg]} { + # Already done. + return + } + + if {[$ts get definition] eq "HD"} { + # Cannot extract a useful MP3 from a HD recording. + ::auto::log " $file - High definition." 2 + return + } + + if {[queue check $ts mpg]} { + ::auto::log " $file - Already queued." 2 + return + } + + # Enqueue file + queue insert $ts mpg + ::auto::log " $file - Queued for mpg." 0 +} + +proc ::mpg::directory {dir} { + ::auto::log "MPG: \[$dir]" 2 + ::auto::direntries $dir ::mpg::ts +} + +proc ::mpg::rundir {dir} { + set sup [::auto::autoflagscanup $dir mpg] + if {$sup == -1} { + log "Encountered special directory." 2 + return + } + ::auto::autoflagscan $dir mpg ::mpg::directory 0 $sup +} + +proc ::mpg::run {} { + ::auto::autoflagscan $::auto::root mpg ::mpg::directory +} + +::auto::register mpg 300 + diff --git a/webif/lib/auto/mpg/queue.hook b/webif/lib/auto/mpg/queue.hook new file mode 100644 index 00000000..db9bce24 --- /dev/null +++ b/webif/lib/auto/mpg/queue.hook @@ -0,0 +1,56 @@ + +proc ::mpg::dequeue {q ts} { + namespace import ::auto::log + + set tmp $::auto::tmp + + set file [file rootname [$ts get file]] + + if {[file exists $file.mpg]} { + return {"OK" "Already done"} + } + + if {[$ts flag "ODEncrypted"]} { + return {"DEFER" "Not decrypted"} + } + + if {[$ts get definition] eq "HD"} { + # Cannot extract a useful MPG from a HD recording. + return {"FAILED" "Cannot process high-definition recording"} + } + + if {[::auto::inuse $ts]} { + return {"DEFER" "Recording in use"} + } + + system startop mpg [$ts get file] + ::auto::dsc [$ts size] + + ::auto::startclock + + log " MPG: $file" 0 + log " Converting..." 0 + if {[catch { + foreach line [split \ + [exec nice -n 19 /mod/bin/ffmpeg -y -benchmark -v 0 \ + -i $file.ts \ + -map 0:0 -map 0:1 \ + -vcodec copy -acodec copy $tmp/mpg.mpg] "\n"] { + log $line 0 + } + } msg]} { + system notify "$file - auto-mpg - error $msg." + system endop mpg + return [list "FAILED" $msg] + } + + # Move the MPG into the local directory + file rename $tmp/mpg.mpg $file.mpg + set summary [::auto::endclock [$ts size]] + system endop mpg + + return [list "OK" $summary] +} + +::auto::register mpg 300 + diff --git a/webif/lib/auto/scan b/webif/lib/auto/scan new file mode 100755 index 00000000..89463b2d --- /dev/null +++ b/webif/lib/auto/scan @@ -0,0 +1,362 @@ +#!/mod/bin/jimsh + +source /mod/webif/lib/setup +require system.class settings.class ts.class rsv.class browse.class \ + queue.class \ + lock plugin safe_delete pretty_size + +source /mod/webif/lib/auto/util.jim + +set ::auto::settings [settings] +set ::auto::loglevel [$::auto::settings autolog] + +set ::auto::prelocked 0 +set ::auto::earlyexit 0 +set ::auto::force 0 + +while {[llength $argv]} { + switch -- [lindex $argv 0] { + -d { + set ::auto::loglevel 2 + set ::auto::logfd stdout + } + -f { + set ::auto::force 1 + } + -prelocked { + set ::auto::prelocked 1 + } + -logprefix { + set argv [lrange $argv 1 end] + if {[llength $argv]} { + set ::auto::logprefix [lindex $argv 0] + } + } + -test { + set ::auto::earlyexit 1 + } + default { + # Pass to rest of script. + break + } + } + set argv [lrange $argv 1 end] +} + +# Acquire lock +if {$::auto::logfd ne "unset"} { + puts $::auto::logfd "Acquiring lock..." +} +if {!$::auto::prelocked && ![acquire_lock webif_autoscan]} { + if {$::auto::loglevel > 1} { + system plog auto "Could not acquire lock." + } + puts "Could not acquire exclusive lock, terminating." + exit +} + +::auto::loginit + +###################################################################### +# Determine if it's time to run + +if {[system uptime] < 180} { + ::auto::log "Aborting, system has just booted." 2 + exit +} + +::auto::dsc +::auto::oktorun +if {!$::auto::force} { + set autofreq [$::auto::settings autofreq] + if {$autofreq == 0} { set autofreq 20 } + + set timesincelast $(([clock seconds] - [$::auto::settings autolast]) / 60) + if {$timesincelast < $autofreq} { + ::auto::log "Aborting, not yet time to run." 2 + ::auto::log " elapsed (minutes): $timesincelast (<$autofreq)" 2 + exit + } +} + +if {$::auto::earlyexit} { + if {!$::auto::force} { + $::auto::settings autolast [clock seconds] + } + puts "Early exit." + exit +} + +######################################################################### +# Initialisation + +set scanstart [clock milliseconds] +::auto::log "Auto processing starting" + +::auto::tmpdir "webif_auto" + +if {[system pkginst undelete]} { + set ::auto::dustbin "[system dustbin]" +} else { + set ::auto::dustbin "" +} + +set ::auto::root [system mediaroot] +file stat "$::auto::root/" rootstat +set ::auto::rootdev $rootstat(dev) + +######################################################################### +# Utility functions + +set ::auto::recalcdirs {} + +proc ::auto::recalcdir {dir} { + variable recalcdirs + ladd recalcdirs $dir +} + +proc ::auto::direntries {dir callback} { + foreach entry [readdir -nocomplain $dir] { + if {![string match {*.ts} $entry]} continue + if {[catch {set ts [ts fetch "$dir/$entry"]}]} continue + if {$ts == 0} continue + $callback $ts + } +} + +proc ::auto::autoflagscan {dir attr callback {recurse 1} {force 0} {seen {}}} \ + {{indent 0} {forceflag ""}} { + variable dustbin + variable rootdev + + incr indent 2 + + log "[string repeat " " $indent]\[$dir]" 2 + + if {$dir eq $dustbin} { + log "[string repeat " " $indent]Dustbin, skipping." 2 + incr indent -2 + return + } + + file stat "$dir/" st + + if {[specialdir $dir]} { + # Special folder + if {$st(dev) != $rootdev} { + log "[string repeat " " $indent\ + ]Special folder on different device, skipping." 2 + incr indent -2 + return + } + if {$force} { + set force 0 + set forceflag "" + log "[string repeat " " $indent\ + ]Special folder, overriding recursion." 2 + } + } + + # Already seen + set key "$st(dev):$st(ino)" + if {$key in $seen} { + log "[string repeat " " $indent]Already seen, skipping." 2 + incr indent -2 + return + } + lappend seen $key + + # Recursion + if {!$force && [file exists "$dir/.auto${attr}r"]} { + log "[string repeat " " $indent] (R)" 2 + set force 1 + set forceflag "$dir/.auto${attr}r" + } + + oktorun + dsc + + if {$force || [file exists "$dir/.auto$attr"]} { + $callback $dir + } + + foreach entry [readdir -nocomplain $dir] { + if {$recurse && [file isdirectory "$dir/$entry"]} { + autoflagscan "$dir/$entry" \ + $attr $callback $recurse $force \ + $seen + file stat "$dir/$entry" st + set key "$st(dev):$st(ino)" + lappend seen $key + } + } + + incr indent -2 +} + +proc ::auto::flagscan {dir flag callback {seen {}}} { + variable dustbin + + if {$dir eq $dustbin} return + + if {[specialdir $dir]} return + + file stat $dir st + set key "$st(dev):$st(ino)" + if {$key in $seen} { + log "Already seen $dir ($key)" 2 + return + } + lappend seen $key + + if {[file exists "$dir/.$flag"]} { $callback $dir } + + foreach entry [readdir -nocomplain $dir] { + if {[file isdirectory "$dir/$entry"]} { + flagscan "$dir/$entry" $flag $callback $seen + + file stat "$dir/$entry" st + set key "$st(dev):$st(ino)" + lappend seen $key + } + } +} + +###################################################################### +# Plugin registration + +set ::auto::plugins {} + +proc ::auto::register {plugin {priority 500} {fn ""}} { + variable plugins + + if {$fn eq ""} { set fn "::${plugin}::run" } + lappend plugins [list $plugin $fn $priority] + log "Registered $plugin with priority $priority ($fn)" +} + +# Backwards compatibility with legacy plugins +set ::auto::legacy {} +proc register {type fn {priority 50}} { + set module [lindex [split $fn :] 2] + + switch -- $type { + predecryptscan { incr priority 600 } + postdecryptscan { incr priority 500 } + prededupscan { incr priority 800 } + postdedupscan { incr priority 700 } + preshrinkscan { incr priority 400 } + postshrinkscan { incr priority 300 } + prempgscan { incr priority 300 } + postmpgscan { incr priority 200 } + premp3scan { incr priority 300 } + postmp3scan { incr priority 200 } + preexpirescan { incr priority 900 } + postexpirescan { incr priority 800 } + postdecryptsingledir { + ::auto::log \ + "Mapping ::${module}::rundir on to $fn" 1 + alias ::${module}::rundir $fn + return + } + default { + ::auto::log \ + "Legacy registration ignored for $type $fn" 1 + return + } + } + ::auto::register $module $priority $fn + lappend ::auto::legacy $module +} + +###################################################################### +# Load plugins + +# Bundled +eval_plugins auto 1 "" /mod/webif/lib/auto + +# Third party + +# temporary for legacy plugins +set settings $::auto::settings +set root $::auto::root +eval_plugins auto 1 +unset settings root + +###################################################################### +# Set expected variables for legacy plugins + +if {[llength $::auto::legacy]} { + ::auto::log "Legacy plugins in use - $::auto::legacy" 1 + + set settings $::auto::settings + set root $::auto::root + alias log ::auto::log + alias scan_run ::auto::flagscan + alias scanup ::auto::autoflagscanup +} + +###################################################################### +# Run plugins + +set ::auto::orderedplugins [\ + lsort -index end -decreasing -integer $::auto::plugins] + +set __dummy "" +proc ::auto::runplugin {fn {_plugin ""} args} { + variable orderedplugins + variable legacy + + foreach p $orderedplugins { + lassign $p plugin xfn priority + + if {$_plugin ne "" && $plugin ne $_plugin} continue + + if {$fn eq "run"} { + set rfn $xfn + } else { + set rfn "::${plugin}::$fn" + } + + if {[exists -proc $rfn] || [exists -alias $rfn]} { + set st [clock milliseconds] + log [string repeat * 56] 2 + log "*********> $rfn (Priority $priority)" 2 + if {$plugin in $legacy && $fn ne "rundir"} { + set call [list $rfn __dummy] + } else { + set call [list $rfn $args] + } + if {[catch {uplevel 1 {*}$call} msg]} { + log "$rfn: $msg" 0 + } + log "<********* $rfn ([elapsed $st] seconds)" 2 + } + } +} + +set ::auto::passes {init run cleanup} +if {[lindex $argv 0] eq "-singledir"} { + foreach dir [lrange $argv 1 end] { + ::auto::runplugin rundir "" $dir + } +} elseif {[llength $argv] > 0} { + foreach pass $::auto::passes { + foreach arg $argv { ::auto::runplugin $pass $arg } + } +} else { + foreach pass $::auto::passes { + ::auto::runplugin $pass + } + $::auto::settings autolast [clock seconds] +} + +if {!$::auto::prelocked} { release_lock webif_autoscan } + +foreach dir $::auto::recalcdirs { + ::auto::log "Running unwatched recalculation for $dir" 2 + ts resetnew $dir +} + +::auto::log "Auto processing completed in [::auto::elapsed $scanstart] seconds." + diff --git a/webif/lib/auto/shrink/auto.hook b/webif/lib/auto/shrink/auto.hook new file mode 100755 index 00000000..3ea79a97 --- /dev/null +++ b/webif/lib/auto/shrink/auto.hook @@ -0,0 +1,57 @@ + +proc ::shrink::ts {ts} { + set file [$ts get file] + + if {[$ts flag "Shrunk"]} { + ::auto::log " $file - already shrunk." 2 + return + } + + if {[queue check $ts shrink]} { + ::auto::log " $file - Already queued." 2 + return + } + + if {[catch { + set perc [exec /mod/bin/stripts -aq [file rootname $file]] + } msg]} { + ::auto::log " Error: $msg" 0 + return + } + if {[string match {*%} $perc]} { + set perc [string range $perc 0 end-1] + } else { + set perc 0 + } + + if {$perc == 0} { + ::auto::log " $file - already shrunk." 2 + $ts set_shrunk + return + } + + # Enqueue file + queue insert $ts shrink + ::auto::log " $file - Queued for shrink." 0 +} + +proc ::shrink::directory {dir} { + ::auto::log "SHRINK: \[$dir]" 2 + ::auto::direntries $dir ::shrink::ts +} + +proc ::shrink::rundir {dir} { + set sup [::auto::autoflagscanup $dir shrink] + if {$sup == -1} { + log "Encountered special directory." 2 + return + } + ::auto::autoflagscan $dir shrink ::shrink::directory 0 $sup +} + +proc ::shrink::run {} { + ::auto::autoflagscan $::auto::root shrink ::shrink::directory +} + +::auto::register shrink 400 + diff --git a/webif/lib/auto/shrink/queue.hook b/webif/lib/auto/shrink/queue.hook new file mode 100644 index 00000000..972a58a5 --- /dev/null +++ b/webif/lib/auto/shrink/queue.hook @@ -0,0 +1,98 @@ + +proc ::shrink::dequeue {q ts} { + global tsgroup + namespace import ::auto::log + + set tmp $::auto::tmp + set file [$ts get file] + + if {[$ts flag "Shrunk"]} { + return {"OK" "Already shrunk"} + } + + # If the directory is flagged for decryption as well as shrink + # then check to see if decryption has already occured. If not, + # defer the shrink for now. + set dir [file dirname $file] + if {[file exists "$dir/.autodecrypt"] || + [::auto::autoflagscanup $dir decrypt] == 1} { + log " $dir is also set for decryption." 2 + if {[$ts flag "ODEncrypted"]} { + return {"DEFER" "Deferring shrink until decrypted"} + } + } + + set file [file rootname [$ts get file]] + + if {[::auto::inuse $ts]} { + return {"DEFER" "Recording in use"} + } + + if {[catch { + set perc [exec /mod/bin/stripts -aq $file] + } msg]} { + return [list "FAILED" $msg] + } + if {[string match {*%} $perc]} { + set perc [string range $perc 0 end-1] + } else { + set perc 0 + } + + if {$perc == 0} { + log " $file - already shrunk." 2 + $ts set_shrunk + return {"OK" "Already shrunk"} + } + set size [$ts size] + ::auto::dsc $size + system startop shrink [$ts get file] + ::auto::startclock + log " SHRINK: $file" 0 + log " Estimate $perc% saving." 0 + log " Shrinking..." 0 + if {[catch { + foreach line [split \ + [exec nice -n 19 /mod/bin/stripts -q $file $tmp/shrunk] \ + "\n"] { + set stats [string trim $line] + log $stats 0 + } + } msg]} { + log "Error during shrink: $msg" 0 + system notify "$file - auto-shrink - error $msg." + system endop shrink + return [list "FAILED" $msg] + } + + # The following steps are structured to minimise the risk of + # things being left in an inconsistent state if the system goes + # into standby. Renames within the same filesystem are very + # quick so the risk is small, but even so... + + # Move the shrunken version back to the local directory. + foreach f [glob "$tmp/shrunk.*"] { + set ext [file extension $f] + file rename $f "${file}_shrunk${ext}" + # Set the file time to match the old file + file touch "${file}_shrunk${ext}" [$ts get file] + } + + # Remove the old recording (-> bin if undelete is installed) + safe_delete [$ts get file] "webif_autoshrink" + + # Finally, rename the shrunken recording again. + foreach ext $tsgroup { + set f "${file}_shrunk.$ext" + if {[file exists $f]} { + file rename $f "${file}.$ext" + } + } + $ts set_shrunk + set summary [::auto::endclock $size] + system endop shrink + return [list "OK" "$summary $stats"] +} + +::auto::register shrink 400 + diff --git a/webif/lib/auto/util.jim b/webif/lib/auto/util.jim new file mode 100644 index 00000000..7e150159 --- /dev/null +++ b/webif/lib/auto/util.jim @@ -0,0 +1,159 @@ + +set ::auto::logfile "/mod/tmp/auto.log" +set ::auto::logfd "unset" +set ::auto::logprefix "" +set ::auto::loglevel 1 + +proc ::auto::loginit {} { + + # Rotate log file if large enough. + if {[file exists $::auto::logfile] && + [file size $::auto::logfile] > 2097152} { + file copy -force $::auto::logfile "/mod/tmp/auto_old.log" + file delete $::auto::logfile + } + + # Open log file + if {$::auto::logfd eq "unset"} { + set ::auto::logfd [open "/mod/tmp/auto.log" "a+"] + } +} + +proc ::auto::log {msg {level 1}} { + variable loglevel + + if {$level > $loglevel} return + + variable logfd + variable logprefix + + puts $logfd "[system logtimestamp] - $logprefix$msg" + flush $logfd +} + +proc ::auto::tmpdir {dir} { + set ::auto::tmp "/mod/tmp/$dir" + if {![file exists $::auto::tmp]} { + if {[catch {file mkdir $::auto::tmp} msg]} { + log "Cannot create temporary directory -"\ + " $::auto::tmp ($msg)" 0 + exit + } + } elseif {![file isdirectory $::auto::tmp]} { + log "Cannot create temporary directory -"\ + " $::auto::tmp (file exists)" 0 + exit + } + + # Clean-up the temporary directory + foreach file [readdir -nocomplain $::auto::tmp] { + file tdelete "$::auto::tmp/$file" + } +} + +proc ::auto::oktorun {} { + variable settings + + set now [clock seconds] + + # Ongoing scheduled recording + if {[$settings noautorec]} { + # Use redring helper files if available. + set recs [glob -nocomplain /tmp/.rec*] + set events [rsv list tbl_reservation \ + " where ersvtype = 3 + and nsttime - ulPreOffset <= $now + and nsttime + nduration + ulPostOffset >= $now "] + if {[llength $recs] || [llength $events]} { + log "Aborted, recording in progress." 2 + exit + } + } + + # Imminent recording + if {[$settings noautorecimm]} { + set autorecperiod $([$settings autorecperiod] * 60) + set events [rsv list tbl_reservation \ + " where ersvtype = 3 + and nsttime - $now < $autorecperiod + and nsttime > $now "] + if {[llength $events]} { + set event [lindex $events 0] + log [concat \ + "Aborted, will record '[$event name]' " \ + "on [$event channel_name] at " \ + "[clock format [$event get nsttime] -format {%H:%M}]" \ + ] 2 + exit + } + } + + # Non-running hours + set hour $([clock format [clock seconds] -format %H] + 0) + if {$hour in [$settings noautohours]} { + log "Aborted, not running during hour $hour" 2 + exit + } +} + +proc ::auto::dsc {{size 0}} { + set free [system diskfree] + + # Required disk space is 1GiB + 3 times the file size. + set req $($size * 3 + 1073741824) + + if {$free < $req} { + log "Insufficient disk space. Require=$req, Free=$free" 0 + exit + } +} + +proc ::auto::inuse {ts} { + set retries 5 + while {$retries > 0 && [$ts inuse]} { + log "inuse($retries) - [$ts get file]" + incr retries -1 + sleep 1 + } + return [$ts inuse] +} + +proc ::auto::specialdir {dir} { + if {[string match {\[*} [string trimleft [file tail $dir]]]} { + return 1 + } + return 0 +} + +proc ::auto::elapsed {start} { + return $(([clock milliseconds] - $start) / 1000.0) +} + +proc ::auto::startclock {} { + set ::startclock_s [clock milliseconds] +} + +proc ::auto::endclock {size} { + set el [elapsed $::startclock_s] + set rate $($size / $el) + return "[pretty_size $size] in $el seconds - [pretty_size $rate]/s" +} + +proc ::auto::autoflagscanup {dir flag} { + variable root + + set rl [string length $root] + while {[string length $dir] >= $rl} { + if {[specialdir $dir]} { + return -1 + } + if {[file exists "$dir/.auto${flag}r"]} { + log "scanup: Found ${flag}r ($dir)" 2 + return 1 + } + set dir [file dirname $dir] + } + + return 0 +} + diff --git a/webif/lib/bin/auto b/webif/lib/bin/auto index 1416d5c4..26a698eb 100755 --- a/webif/lib/bin/auto +++ b/webif/lib/bin/auto @@ -1,1005 +1,1009 @@ #!/mod/bin/jimsh -source /mod/webif/lib/setup -require lock system.class ts.class pretty_size browse.class \ - safe_delete settings.class plugin queue.class rsv.class +exit -set settings [settings] -set loglevel [$settings autolog] -set audiomp3 [$settings audiomp3] - -set modules {decrypt dedup shrink mpg mp3 expire} - -set queue_actions { - decrypt do_decrypt - strip do_shrink - shrink do_shrink - mpg do_mpg - mp3 do_mp3 -} - -set root [system mediaroot] - -set logfile "/mod/tmp/auto.log" - -set logfd "unset" -set logprefix "" -set prelocked 0 -set earlyexit 0 -while {[llength $argv]} { - switch -- [lindex $argv 0] { - -d { - set loglevel 2 - set logfd stdout - } - -prelocked { - set prelocked 1 - } - -logprefix { - set argv [lrange $argv 1 end] - if {[llength $argv]} { - set logprefix [lindex $argv 0] - } - } - -test { - set earlyexit 1 - } - default { - # Pass to rest of script. - break - } - } - set argv [lrange $argv 1 end] -} - -set manualrun $([llength $argv] > 0) -if {$loglevel > 1} { - if {$logfd ne "unset"} { - puts $logfd "Manual run: $manualrun" - } -} +# No longer used, see /mod/webif/lib/auto/ +###source /mod/webif/lib/setup +###require lock system.class ts.class pretty_size browse.class \ +### safe_delete settings.class plugin queue.class rsv.class +### +###set settings [settings] +###set loglevel [$settings autolog] +###set audiomp3 [$settings audiomp3] +### +###set modules {decrypt dedup shrink mpg mp3 expire} +### +###set queue_actions { +### decrypt do_decrypt +### strip do_shrink +### shrink do_shrink +### mpg do_mpg +### mp3 do_mp3 +###} +### +###set root [system mediaroot] +### +###set logfile "/mod/tmp/auto.log" +### +###set logfd "unset" +###set logprefix "" +###set prelocked 0 +###set earlyexit 0 +###while {[llength $argv]} { +### switch -- [lindex $argv 0] { +### -d { +### set loglevel 2 +### set logfd stdout +### } +### -prelocked { +### set prelocked 1 +### } +### -logprefix { +### set argv [lrange $argv 1 end] +### if {[llength $argv]} { +### set logprefix [lindex $argv 0] +### } +### } +### -test { +### set earlyexit 1 +### } +### default { +### # Pass to rest of script. +### break +### } +### } +### set argv [lrange $argv 1 end] +###} +### +###set manualrun $([llength $argv] > 0) +###if {$loglevel > 1} { +### if {$logfd ne "unset"} { +### puts $logfd "Manual run: $manualrun" +### } +###} +### +############################################################################ +### +#### Acquire lock +###if {$logfd ne "unset"} { +### puts $logfd "Acquiring lock..." +###} +###if {!$prelocked && ![acquire_lock webif_auto]} { +### if {$loglevel > 1} { +### system plog auto "Could not acquire lock." +### } +### puts "Could not acquire exclusive lock, terminating." +### exit +###} +### +#### Rotate log file if large enough. +###if {[file exists $logfile] && [file size $logfile] > 2097152} { +### file copy -force $logfile "/mod/tmp/auto_old.log" +### file delete $logfile +###} +### +#### Open log file +###if {$logfd eq "unset"} { +### set logfd [open "/mod/tmp/auto.log" "a+"] +###} +### +############################################################################ +### +###proc log {msg {level 1}} { +### if {$level > $::loglevel} return +### puts $::logfd "[system logtimestamp] - $::logprefix$msg" +### flush $::logfd +###} +### +###proc oktorun {} { +### global settings +### +### set now [clock seconds] +### +### # Ongoing scheduled recording +### if {[$settings noautorec]} { +### # Use redring helper files if available. +### set recs [glob -nocomplain /tmp/.rec*] +### set events [rsv list tbl_reservation \ +### " where ersvtype = 3 +### and nsttime - ulPreOffset <= $now +### and nsttime + nduration + ulPostOffset >= $now "] +### if {[llength $recs] || [llength $events]} { +### log "Aborted, recording in progress." 2 +### exit +### } +### } +### +### # Imminent recording +### if {[$settings noautorecimm]} { +### set autorecperiod $([$settings autorecperiod] * 60) +### set events [rsv list tbl_reservation \ +### " where ersvtype = 3 +### and nsttime - $now < $autorecperiod +### and nsttime > $now "] +### if {[llength $events]} { +### set event [lindex $events 0] +### log [concat \ +### "Aborted, will record '[$event name]' " \ +### "on [$event channel_name] at " \ +### "[clock format [$event get nsttime] -format {%H:%M}]" \ +### ] 2 +### exit +### } +### } +### +### # Non-running hours +### set hour $([clock format [clock seconds] -format %H] + 0) +### if {$hour in [$settings noautohours]} { +### log "Aborted, not running during hour $hour" 2 +### exit +### } +###} +### +###proc dsc {{size 0}} { +### set free [system diskfree] +### +### # Required disk space is 1GiB + 3 times the file size. +### set req $($size * 3 + 1073741824) +### +### if {$free < $req} { +### log "Insufficient disk space. Require=$req, Free=$free" 0 +### exit +### } +###} +### +#### Determine if it's time to run +### +###if {[system uptime] < 180} { +### log "Aborting, system has just booted." 2 +### exit +###} +### +###dsc +###oktorun +###if {!$manualrun} { +### set autofreq [$settings autofreq] +### if {$autofreq == 0} { set autofreq 20 } +### +### set timesincelast $(([clock seconds] - [$settings autolast]) / 60) +### if {![queue size] && $timesincelast < $autofreq} { +### log "Aborting, not yet time to run." 2 +### log " elapsed (minutes): $timesincelast (<$autofreq)" 2 +### exit +### } +###} +### +###if {$earlyexit} { +### if {!$manualrun} { +### $settings autolast [clock seconds] +### } +### puts "Early exit." +### exit +###} +### +############################################################################ +### +###proc inuse {ts} { +### set retries 5 +### while {$retries > 0 && [$ts inuse]} { +### log "inuse($retries) - [$ts get file]" +### incr retries -1 +### sleep 1 +### } +### return [$ts inuse] +###} +### +###proc specialdir {dir} { +### if {[string match {\[*} [string trimleft [file tail $dir]]]} { +### return 1 +### } +### return 0 +###} +### +###proc elapsed {start} { +### return $(([clock milliseconds] - $start) / 1000.0) +###} +### +###proc startclock {} { +### set ::startclock_s [clock milliseconds] +###} +### +###proc endclock {size} { +### set el [elapsed $::startclock_s] +### set rate $($size / $el) +### return "[pretty_size $size] in $el seconds - [pretty_size $rate]/s" +###} +### +###foreach mod $modules { +### set "hook_pre${mod}scan" {} +### set "hook_pre$mod" {} +### set "hook_post$mod" {} +### set "hook_post${mod}scan" {} +### set "hook_pre${mod}singledir" {} +### set "hook_post${mod}singledir" {} +###} +### +###proc register {type fn {priority 50}} { +### global "hook_$type" +### if {[info exists "hook_$type"]} { +### lappend "hook_$type" [list $fn $priority] +### log "Registered $fn for $type hook with priority $priority." +### } else { +### log "Unknown hook hook_$type" 0 +### } +###} +### +###proc register_queue {action function} { +### set ::queue_actions($action) $function +###} +### +###proc runplugin {name {ts 0}} { +### set var "hook_$name" +### global $var +### foreach p [lsort -index end -decreasing -integer [subst $$var]] { +### lassign $p fn priority +### log "Running $name plugin $fn (priority $priority) \[$ts]" 2 +### if {[catch {uplevel 1 [list $fn $ts]} msg]} { +### log "Plugin error: $msg" 0 +### foreach {proc file line} [info stacktrace] { +### log " @ $file:$proc:$line" 0 +### } +### } +### log "Completed $name plugin $fn" 2 +### } +###} +### +############################################################################ +### +###eval_plugins auto 1 +### +###set scanstart [clock milliseconds] +###log "-------------------------------------------------------" +### +#### dlnastatus is relatively expensive so it is checked once globally at +#### the start and then if the server is not listening then no decrypt +#### operations will be attempted for this run, even if the server starts +#### up halfway through. Otherwise the server is checked for every decryption +#### and if it goes away then decryption will not be attempted for the rest +#### of the run. +###if {[system dlnastatus]} { +### set dlnaok 1 +### log "DLNA Server is running." 2 +###} else { +### set dlnaok 0 +### log "DLNA Server is NOT running." 2 +### if {[system instandby]} { +### log "System is in standby." 2 +### } elseif {[system param DMS_START_ON] ne "1"} { +### log "DLNA is disabled." 2 +### } else { +### log "Giving DLNA more time to start..." 2 +### loop x 0 10 { +### sleep 10 +### if {[system dlnastatus]} { +### set dlnaok 1 +### log "DLNA Server is running." 2 +### break +### } +### log "DLNA Server not yet running, waiting..." 2 +### } +### } +###} +### +###log "Auto processing starting, DLNA server status: $dlnaok" +### +###dsc +### +###set tmp "/mod/tmp/webif_auto" +###if {![file exists $tmp]} { +### if {[catch {file mkdir $tmp} msg]} { +### log "Cannot create temporary directory - $tmp ($msg)" 0 +### exit +### } +###} elseif {![file isdirectory $tmp]} { +### log "Cannot create temporary directory - $tmp (file exists)" 0 +### exit +###} +### +#### Clean-up the temporary directory +###foreach file [readdir -nocomplain $tmp] { file tdelete "$tmp/$file" } +### +###if {[system pkginst undelete]} { +### set dustbin "[system dustbin]" +###} else { +### set dustbin "" +###} +### +###log "Dustbin: $dustbin" 2 +### +###set recalc 0 +### +###proc dorecalc {dir} { +### global recalc +### if {!$recalc} return +### log "Running unwatched recalculation for $dir" 2 +### ts resetnew $dir +### incr recalc -1 +###} +### +###proc dedup {dir {flag ""}} { +### log "DEDUP: \[$dir]" 2 +### loop i 0 1 { +### foreach line [split \ +### [exec /mod/webif/html/dedup/dedup -yes -auto $dir] "\n"] { +### log $line 2 +### } +### } +### incr ::recalc +###} +### +###proc do_shrink {ts} { +### global tmp dustbin tsgroup processed_files +### set file [$ts get file] +### +### if {[$ts flag "Shrunk"]} { +### log " $file - already shrunk." 2 +### return +### } +### +### # If the directory is flagged for decryption as well as shrink +### # then check to see if decryption has already occured. If not, +### # defer the shrink for now. +### set dir [file dirname $file] +### if {[file exists "$dir/.autodecrypt"] || [scanup $dir decrypt] == 1} { +### log " $dir is also set for decryption." 2 +### if {[$ts flag "ODEncrypted"]} { +### log " $file - deferring shrink until decrypted." +### return +### } +### } +### +### set file [file rootname [$ts get file]] +### +### if {[inuse $ts]} { +### log " $file - in use." 2 +### return +### } +### +### if {[catch { +### set perc [exec /mod/bin/stripts -aq $file] +### } msg]} { +### log " Error: $msg" 0 +### return +### } +### if {[string match {*%} $perc]} { +### set perc [string range $perc 0 end-1] +### } else { +### set perc 0 +### } +### +### if {$perc == 0} { +### log " $file - already shrunk." 2 +### $ts set_shrunk +### return +### } +### oktorun +### set size [$ts size] +### dsc $size +### system startop shrink [$ts get file] +### runplugin preshrink $ts +### startclock +### log " SHRINK: $file" 0 +### log " Estimate $perc% saving." 0 +### log " Shrinking..." 0 +### if {[catch { +### foreach line [split \ +### [exec nice -n 19 /mod/bin/stripts -q $file $tmp/shrunk] \ +### "\n"] { +### log $line 0 +### } +### } msg]} { +### log "Error during shrink: $msg" 0 +### system notify "$file - auto-shrink - error $msg." +### system endop shrink +### return +### } +### +### # The following steps are structured to minimise the risk of +### # things being left in an inconsistent state if the system goes +### # into standby. Renames within the same filesystem are very +### # quick so the risk is small, but even so... +### +### # Move the shrunken version back to the local directory. +### foreach f [glob "$tmp/shrunk.*"] { +### set ext [file extension $f] +### file rename $f "${file}_shrunk${ext}" +### # Set the file time to match the old file +### file touch "${file}_shrunk${ext}" [$ts get file] +### } +### +### # Remove the old recording (-> bin if undelete is installed) +### safe_delete [$ts get file] "webif_autoshrink" +### +### # Finally, rename the shrunken recording again. +### foreach ext $tsgroup { +### set f "${file}_shrunk.$ext" +### if {[file exists $f]} { +### file rename $f "${file}.$ext" +### } +### } +### $ts set_shrunk +### log "Done... [endclock $size]" 0 +### lappend processed_files [$ts get file] +### runplugin postshrink $ts +### system endop shrink +###} +### +###proc do_decrypt {ts} { +### global tmp dustbin processed_files +### +### set file [$ts get file] +### set rfile [file rootname $file] +### set bfile [file tail $file] +### +### if {![$ts flag "ODEncrypted"]} { +### log " $file - Already decrypted." 2 +### return +### } +### +### if {[$ts flag "Encrypted"]} { +### log " $file - Protected (Enc flag)." 2 +### return +### } +### +### if {![system dlnastatus]} { +### log " $file - DLNA Server not running." 2 +### set ::dlnaok 0 +### return +### } +### +### if {[inuse $ts]} { +### log " $file - In use." +### return +### } +### +### # Check that the file is not already decrypted by analysing it. +### set anencd [exec /mod/bin/stripts -qE $rfile] +### if {$anencd != "1"} { +### log " $file - already decrypted but the HMT flag is wrong." 0 +### # Fix... +### exec /mod/webif/lib/bin/fixencflags $file +### return +### } +### +### lassign [$ts dlnaloc "127.0.0.1"] url +### if {$url ne ""} { +### log " $file - has been indexed." +### set helper 0 +### } else { +### log " $file - Not yet indexed, trying helper." +### if {[catch { +### lassign [system dlnahelper [file normalize $file]] url +### } msg]} { +### log " $file - $msg" +### } +### if {$url eq ""} { +### log " $file - Can't use helper." +### return +### } +### set helper 1 +### } +### +### # Perform the decryption by requesting the file from the DLNA server. +### oktorun +### set size [$ts size] +### dsc $size +### runplugin predecrypt $ts +### system startop decrypt $file +### startclock +### log " DECRYPT: $rfile" 0 +### log " DLNA: $url" 0 +### exec wget -O "$tmp/$bfile" $url +### +### # Release the helper lock once finished. +### if {$helper} { system dlnahelper -release } +### +### if {[file size $file] != [file size "$tmp/$bfile"]} { +### log " $file - File size mismatch." 0 +### file tdelete "$tmp/$bfile" +### system endop decrypt +### return +### } +### +### # Check if the file is in use. It is possible that the file is +### # now being played even though it was free when decryption started. +### if {[inuse $ts]} { +### log " $file - In use." +### file tdelete "$tmp/$bfile" +### system endop decrypt +### return +### } +### +### # Copy the HMT file over for stripts +### set thmt "$tmp/[file rootname $bfile].hmt" +### file copy "$rfile.hmt" $thmt +### # Check that the file is no longer encrypted by analysing it. +### set anencd [exec /mod/bin/stripts -qE "$tmp/[file rootname $bfile]"] +### file delete $thmt +### +### if {$anencd != "0"} { +### log " $file - File did not decrypt properly." 0 +### file tdelete "$tmp/$bfile" +### if {[clock seconds] - [$ts get end] > 3 * 3600} { +### system notify "$file - auto-decrypt failed." +### } +### system endop decrypt +### return +### } +### +### # Move the encrypted file out of the way. +### file rename $file "$rfile.encrypted" +### # Move the decrypted copy into place. +### file rename "$tmp/$bfile" $file +### # Set the file time to match the old file +### file touch $file "$rfile.encrypted" +### # Patch the HMT - quickest way to get back to a playable file. +### exec /mod/bin/hmt -encrypted "$rfile.hmt" +### +### log " Removing/binning old copy." 0 +### # Move the old recording to the bin if undelete is installed. +### if {$dustbin ne ""} { +### set bin [_del_bindir $file "webif_autodecrypt"] +### set tail [file tail $rfile] +### file rename "$rfile.encrypted" "$bin/$tail.ts" +### foreach ext {nts hmt thm} { +### if {[file exists "$rfile.$ext"]} { +### file copy $rfile.$ext "$bin/$tail.$ext" +### if {$ext eq "hmt"} { +### # Patch the binned HMT back +### exec /mod/bin/hmt +encrypted \ +### "$bin/$tail.hmt" +### } +### } +### } +### } else { +### file tdelete "$rfile.encrypted" +### } +### log "Done... [endclock $size]" 0 +### lappend processed_files [$ts get file] +### $ts unflag "ODEncrypted" +### runplugin postdecrypt $ts +### system endop decrypt +###} +### +###proc do_mpg {ts} { +### global tmp tsgroup processed_files +### +### set file [file rootname [$ts get file]] +### +### if {[file exists $file.mpg]} { +### # Already done. +### return +### } +### +### if {[$ts flag "ODEncrypted"]} { +### log " $file - Not decrypted." 2 +### return +### } +### +### if {[$ts get definition] eq "HD"} { +### # Cannot extract a useful MPG from a HD recording. +### return +### } +### +### if {[inuse $ts]} { +### log " $file - In use." +### return +### } +### oktorun +### system startop mpg [$ts get file] +### runplugin prempg $ts +### dsc [$ts size] +### +### log " MPG: $file" 0 +### log " Converting..." 0 +### if {[catch { +### foreach line [split \ +### [exec nice -n 19 /mod/bin/ffmpeg -y -benchmark -v 0 \ +### -i $file.ts \ +### -map 0:0 -map 0:1 \ +### -vcodec copy -acodec copy $tmp/mpg.mpg] "\n"] { +### log $line 0 +### } +### } msg]} { +### log "Error during mpg extract: $msg" 0 +### system notify "$file - auto-mpg - error $msg." +### system endop mpg +### return +### } +### +### # Move the MPG into the local directory +### file rename $tmp/mpg.mpg $file.mpg +### lappend processed_files [$ts get file] +### runplugin postmpg $ts +### system endop mpg +###} +### +###proc do_mp3 {ts} { +### global tmp tsgroup processed_files +### +### set file [file rootname [$ts get file]] +### +### if {[file exists $file.mp3]} { +### # Already done. +### return +### } +### +### if {[$ts flag "ODEncrypted"]} { +### log " $file - Not decrypted." 2 +### return +### } +### +### if {[$ts get definition] eq "HD"} { +### # Cannot extract a useful MP3 from a HD recording. +### log " $file - High definition." 2 +### return +### } +### +### if {[inuse $ts]} { +### log " $file - In use." +### return +### } +### oktorun +### system startop mp3 [$ts get file] +### runplugin premp3 $ts +### dsc [$ts size] +### +### log " MP3: $file" 0 +### log " Converting... [$::settings audiomp3descr $::audiomp3]" 0 +### if {[catch { +### set cmd [list nice -n 19 \ +### /mod/bin/ffmpeg -y -benchmark -v 0 -i "$file.ts" \ +### -f mp3 -vn] +### if {!$::audiomp3} { lappend cmd -acodec copy } +### lappend cmd "$tmp/mp3.mp3" +### foreach line [split [exec {*}$cmd] "\n"] { +### log $line 0 +### } +### } msg]} { +### log "Error during mp3 extract: $msg" 0 +### system notify "$file - auto-mp3 - error $msg." +### system endop mp3 +### return +### } +### +### if {[system pkginst id3v2]} { +### log [exec /mod/bin/id3v2 \ +### --song "[$ts get title]" \ +### --comment "[$ts get synopsis]" \ +### --album "[$ts get channel_name]" \ +### --year "[clock format [$ts get start] -format {%Y}]" \ +### "$tmp/mp3.mp3"] 0 +### } +### +### # Move the MP3 into the local directory +### file rename $tmp/mp3.mp3 $file.mp3 +### lappend processed_files [$ts get file] +### runplugin postmp3 $ts +### system endop mp3 +###} +### +###proc entries {dir callback {flag ""}} { +### foreach entry [readdir -nocomplain $dir] { +### if {$flag ne "" && ![file exists $flag]} { +### log "Flag $flag disappeared mid-run, aborting." 0 +### break +### } +### if {![string match {*.ts} $entry]} continue +### if {[catch {set ts [ts fetch "$dir/$entry"]}]} continue +### if {$ts == 0} continue +### $callback $ts +### } +###} +### +###proc shrink {dir {flag ""}} { +### log "SHRINK: \[$dir]" 2 +### entries $dir do_shrink $flag +###} +### +###proc decrypt {dir {flag ""}} { +### log "DECRYPT: \[$dir]" 2 +### if {$::dlnaok} { entries $dir do_decrypt $flag } +###} +### +###proc mpg {dir {flag ""}} { +### log "MPG: \[$dir]" 2 +### entries $dir do_mpg $flag +###} +### +###proc mp3 {dir {flag ""}} { +### log "MP3: \[$dir]" 2 +### entries $dir do_mp3 $flag +###} +### +###proc expire {dir {flag ""}} { +### log "EXPIRE: \[$dir]" 2 +### +### # type 0 keep {} days 2 keepnew 0 +### set ax [dir expiry $dir] +### log " $ax" 2 +### if {![string is integer -strict $ax(keep)]} { set ax(keep) 0 } +### if {![string is integer -strict $ax(days)]} { set ax(days) 0 } +### +### if {!$ax(keep) && !$ax(days)} { +### log "Expire error ($dir): neither keep nor days set ($ax)" 0 +### return +### } +### +### if {$ax(type) eq "0"} { +### set getter [lambda {ts} { list [$ts get end] }] +### } else { +### set getter [lambda {ts} { list [$ts lastmod] }] +### } +### +### set entries [lsort \ +### -command [lambda {a b} {upvar getter g; expr [$g $a] - [$g $b]}]\ +### [lmap i [readdir -nocomplain $dir] { +### if {![string match {*.ts} $i]} continue +### if {[catch {set ts [ts fetch "$dir/$i"]}]} continue +### if {$ts == 0} continue +### list $ts +### }]] +### +### # Entries are now sorted with oldest first, according to the selected +### # criterion. +### +### set num [llength $entries] +### log " Entries: $num" 2 +### foreach e $entries { log " [file tail [$e get file]]" 2 } +### +### set now [clock seconds] +### foreach ts $entries { +### set file [$ts get file] +### if {$num <= $ax(keep)} { +### log " Remaining ($num) <= keep ($ax(keep))" 2 +### break +### } +### log " [file tail $file]" +### if {[inuse $ts]} { +### log " In use, skipping." +### continue +### } +### if {$ax(keepnew) ne "0" && [$ts flag New]} { +### log " Unwatched, skipping." +### continue +### } +### if {$ax(days) > 0} { +### # Calculate age in days +### set age $(($now - [$getter $ts]) / 86400.0) +### log " $age days (cf. $ax(days))" +### if {$age < $ax(days)} { +### log " Too new, skipping." +### continue +### } +### } +### log " Removing." +### runplugin preexpire $ts +### if {![safe_delete $file]} { +### log "Unknown error in safe_delete, aborting." 0 +### break +### } +### log " EXPIRE: removed $file" 0 +### runplugin postexpire $ts +### incr ::recalc +### incr num -1 +### } +###} +### +###proc scan_run {dir flag callback {seen {}}} { +### global dustbin +### +### if {$dir eq $dustbin} return +### +### if {[specialdir $dir]} return +### +### file stat $dir st +### set key "$st(dev):$st(ino)" +### if {$key in $seen} { +### log "Already seen $dir ($key)" 2 +### return +### } +### lappend seen $key +### +### if {[file exists "$dir/.$flag"]} { $callback $dir } +### +### foreach entry [readdir -nocomplain $dir] { +### if {[file isdirectory "$dir/$entry"]} { +### scan_run "$dir/$entry" $flag $callback $seen +### +### file stat "$dir/$entry" st +### set key "$st(dev):$st(ino)" +### lappend seen $key +### } +### } +###} +### +###proc xscan {dir attr {force 0} {recurse 1} {seen {}}} {{indent 0} {forceflag ""}} { +### global dustbin +### +### incr indent 2 +### +### log "[string repeat " " $indent]\[$dir]" 2 +### +### if {$dir eq $dustbin} { +### log "[string repeat " " $indent]Dustbin, skipping." 2 +### incr indent -2 +### return +### } +### +### file stat "$dir/" st +### +### if {[specialdir $dir]} { +### # Special folder +### if {$st(dev) != $::rootdev} { +### log "[string repeat " " $indent]Special folder on different device, skipping." 2 +### incr indent -2 +### return +### } +### if {$force} { +### set force 0 +### set forceflag "" +### log "[string repeat " " $indent]Special folder, overriding recursion." 2 +### } +### } +### +### # Already seen +### set key "$st(dev):$st(ino)" +### if {$key in $seen} { +### log "[string repeat " " $indent]Already seen, skipping." 2 +### incr indent -2 +### return +### } +### lappend seen $key +### +### # Recursion +### if {!$force && [file exists "$dir/.auto${attr}r"]} { +### log "[string repeat " " $indent] (R)" 2 +### set force 1 +### set forceflag "$dir/.auto${attr}r" +### } +### +### oktorun +### dsc +### +### if {$force} { +### $attr $dir $forceflag +### } elseif {[file exists "$dir/.auto$attr"]} { +### $attr $dir "$dir/.auto$attr" +### } +### +### foreach entry [readdir -nocomplain $dir] { +### if {$recurse && [file isdirectory "$dir/$entry"]} { +### xscan "$dir/$entry" $attr $force $recurse $seen +### file stat "$dir/$entry" st +### set key "$st(dev):$st(ino)" +### lappend seen $key +### } +### } +### +### dorecalc $dir +### +### incr indent -2 +###} +### +###proc scanup {dir flag} { +### global root +### +### set rl [string length $root] +### while {[string length $dir] >= $rl} { +### if {[specialdir $dir]} { +### return -1 +### } +### if {[file exists "$dir/.auto${flag}r"]} { +### log "scanup: Found ${flag}r ($dir)" 2 +### return 1 +### } +### set dir [file dirname $dir] +### } +### +### return 0 +###} +### +###proc scansingle {dirs} { +### global modules root +### +### foreach dir $dirs { +### log "Scanning single directory '$dir'" +### foreach arg $modules { +### set st [clock milliseconds] +### set sup [scanup $dir $arg] +### if {$sup == -1} { +### log "Encountered special directory." +### break +### } +### runplugin "pre${arg}singledir" $dir +### xscan $dir $arg $sup 0 +### runplugin "post${arg}singledir" $dir +### log "$arg scan completed in [elapsed $st] seconds." +### } +### } +###} +### +###proc process_queue {} { +### global settings +### log "Processing queue..." 2 +### queue startup [$settings autokeep] +### while {[llength [set q [queue pop]]]} { +### oktorun +### +### if {[catch {set ts [ts fetch [$q get file]]}]} { +### $q update FAILED "Could not open recording." 1 +### continue +### } +### if {$ts == 0} { +### $q update FAILED "Could not open recording." 1 +### continue +### } +### set act [$q get action] +### if {![dict exists $::queue_actions $act]} { +### log "Unknown queue action, $act" +### $q update FAILED "Unknown queue action, $act" 1 +### continue +### } +### set func $::queue_actions($act) +### $q update RUNNING "Started at [clock format [clock seconds]]" +### set start [clock seconds] +### if {[catch {$func $ts} msg]} { +### $q update FAILED "$msg" 1 +### continue +### } +### set elapsed $([clock seconds] - $start) +### $q update COMPLETE \ +### "Completed at [clock format [clock seconds]]" 0 $elapsed +### } +###} +### +###file stat "$root/" rootstat +###set rootdev $rootstat(dev) +###log "Root device: $rootdev" 2 +### ######################################################################### - -# Acquire lock -if {$logfd ne "unset"} { - puts $logfd "Acquiring lock..." -} -if {!$prelocked && ![acquire_lock webif_auto]} { - if {$loglevel > 1} { - system plog auto "Could not acquire lock." - } - puts "Could not acquire exclusive lock, terminating." - exit -} - -# Rotate log file if large enough. -if {[file exists $logfile] && [file size $logfile] > 2097152} { - file copy -force $logfile "/mod/tmp/auto_old.log" - file delete $logfile -} - -# Open log file -if {$logfd eq "unset"} { - set logfd [open "/mod/tmp/auto.log" "a+"] -} - -######################################################################### - -proc log {msg {level 1}} { - if {$level > $::loglevel} return - puts $::logfd "[system logtimestamp] - $::logprefix$msg" - flush $::logfd -} - -proc oktorun {} { - global settings - - set now [clock seconds] - - # Ongoing scheduled recording - if {[$settings noautorec]} { - # Use redring helper files if available. - set recs [glob -nocomplain /tmp/.rec*] - set events [rsv list tbl_reservation \ - " where ersvtype = 3 - and nsttime - ulPreOffset <= $now - and nsttime + nduration + ulPostOffset >= $now "] - if {[llength $recs] || [llength $events]} { - log "Aborted, recording in progress." 2 - exit - } - } - - # Imminent recording - if {[$settings noautorecimm]} { - set autorecperiod $([$settings autorecperiod] * 60) - set events [rsv list tbl_reservation \ - " where ersvtype = 3 - and nsttime - $now < $autorecperiod - and nsttime > $now "] - if {[llength $events]} { - set event [lindex $events 0] - log [concat \ - "Aborted, will record '[$event name]' " \ - "on [$event channel_name] at " \ - "[clock format [$event get nsttime] -format {%H:%M}]" \ - ] 2 - exit - } - } - - # Non-running hours - set hour $([clock format [clock seconds] -format %H] + 0) - if {$hour in [$settings noautohours]} { - log "Aborted, not running during hour $hour" 2 - exit - } -} - -proc dsc {{size 0}} { - set free [system diskfree] - - # Required disk space is 1GiB + 3 times the file size. - set req $($size * 3 + 1073741824) - - if {$free < $req} { - log "Insufficient disk space. Require=$req, Free=$free" 0 - exit - } -} - -# Determine if it's time to run - -if {[system uptime] < 180} { - log "Aborting, system has just booted." 2 - exit -} - -dsc -oktorun -if {!$manualrun} { - set autofreq [$settings autofreq] - if {$autofreq == 0} { set autofreq 20 } - - set timesincelast $(([clock seconds] - [$settings autolast]) / 60) - if {![queue size] && $timesincelast < $autofreq} { - log "Aborting, not yet time to run." 2 - log " elapsed (minutes): $timesincelast (<$autofreq)" 2 - exit - } -} - -if {$earlyexit} { - if {!$manualrun} { - $settings autolast [clock seconds] - } - puts "Early exit." - exit -} - -######################################################################### - -proc inuse {ts} { - set retries 5 - while {$retries > 0 && [$ts inuse]} { - log "inuse($retries) - [$ts get file]" - incr retries -1 - sleep 1 - } - return [$ts inuse] -} - -proc specialdir {dir} { - if {[string match {\[*} [string trimleft [file tail $dir]]]} { - return 1 - } - return 0 -} - -proc elapsed {start} { - return $(([clock milliseconds] - $start) / 1000.0) -} - -proc startclock {} { - set ::startclock_s [clock milliseconds] -} - -proc endclock {size} { - set el [elapsed $::startclock_s] - set rate $($size / $el) - return "[pretty_size $size] in $el seconds - [pretty_size $rate]/s" -} - -foreach mod $modules { - set "hook_pre${mod}scan" {} - set "hook_pre$mod" {} - set "hook_post$mod" {} - set "hook_post${mod}scan" {} - set "hook_pre${mod}singledir" {} - set "hook_post${mod}singledir" {} -} - -proc register {type fn {priority 50}} { - global "hook_$type" - if {[info exists "hook_$type"]} { - lappend "hook_$type" [list $fn $priority] - log "Registered $fn for $type hook with priority $priority." - } else { - log "Unknown hook hook_$type" 0 - } -} - -proc register_queue {action function} { - set ::queue_actions($action) $function -} - -proc runplugin {name {ts 0}} { - set var "hook_$name" - global $var - foreach p [lsort -index end -decreasing -integer [subst $$var]] { - lassign $p fn priority - log "Running $name plugin $fn (priority $priority) \[$ts]" 2 - if {[catch {uplevel 1 [list $fn $ts]} msg]} { - log "Plugin error: $msg" 0 - foreach {proc file line} [info stacktrace] { - log " @ $file:$proc:$line" 0 - } - } - log "Completed $name plugin $fn" 2 - } -} - -######################################################################### - -eval_plugins auto 1 - -set scanstart [clock milliseconds] -log "-------------------------------------------------------" - -# dlnastatus is relatively expensive so it is checked once globally at -# the start and then if the server is not listening then no decrypt -# operations will be attempted for this run, even if the server starts -# up halfway through. Otherwise the server is checked for every decryption -# and if it goes away then decryption will not be attempted for the rest -# of the run. -if {[system dlnastatus]} { - set dlnaok 1 - log "DLNA Server is running." 2 -} else { - set dlnaok 0 - log "DLNA Server is NOT running." 2 - if {[system instandby]} { - log "System is in standby." 2 - } elseif {[system param DMS_START_ON] ne "1"} { - log "DLNA is disabled." 2 - } else { - log "Giving DLNA more time to start..." 2 - loop x 0 10 { - sleep 10 - if {[system dlnastatus]} { - set dlnaok 1 - log "DLNA Server is running." 2 - break - } - log "DLNA Server not yet running, waiting..." 2 - } - } -} - -log "Auto processing starting, DLNA server status: $dlnaok" - -dsc - -set tmp "/mod/tmp/webif_auto" -if {![file exists $tmp]} { - if {[catch {file mkdir $tmp} msg]} { - log "Cannot create temporary directory - $tmp ($msg)" 0 - exit - } -} elseif {![file isdirectory $tmp]} { - log "Cannot create temporary directory - $tmp (file exists)" 0 - exit -} - -# Clean-up the temporary directory -foreach file [readdir -nocomplain $tmp] { file tdelete "$tmp/$file" } - -if {[system pkginst undelete]} { - set dustbin "[system dustbin]" -} else { - set dustbin "" -} - -log "Dustbin: $dustbin" 2 - -set recalc 0 - -proc dorecalc {dir} { - global recalc - if {!$recalc} return - log "Running unwatched recalculation for $dir" 2 - ts resetnew $dir - incr recalc -1 -} - -proc dedup {dir {flag ""}} { - log "DEDUP: \[$dir]" 2 - loop i 0 1 { - foreach line [split \ - [exec /mod/webif/html/dedup/dedup -yes -auto $dir] "\n"] { - log $line 2 - } - } - incr ::recalc -} - -proc do_shrink {ts} { - global tmp dustbin tsgroup processed_files - set file [$ts get file] - - if {[$ts flag "Shrunk"]} { - log " $file - already shrunk." 2 - return - } - - # If the directory is flagged for decryption as well as shrink - # then check to see if decryption has already occured. If not, - # defer the shrink for now. - set dir [file dirname $file] - if {[file exists "$dir/.autodecrypt"] || [scanup $dir decrypt] == 1} { - log " $dir is also set for decryption." 2 - if {[$ts flag "ODEncrypted"]} { - log " $file - deferring shrink until decrypted." - return - } - } - - set file [file rootname [$ts get file]] - - if {[inuse $ts]} { - log " $file - in use." 2 - return - } - - if {[catch { - set perc [exec /mod/bin/stripts -aq $file] - } msg]} { - log " Error: $msg" 0 - return - } - if {[string match {*%} $perc]} { - set perc [string range $perc 0 end-1] - } else { - set perc 0 - } - - if {$perc == 0} { - log " $file - already shrunk." 2 - $ts set_shrunk - return - } - oktorun - set size [$ts size] - dsc $size - system startop shrink [$ts get file] - runplugin preshrink $ts - startclock - log " SHRINK: $file" 0 - log " Estimate $perc% saving." 0 - log " Shrinking..." 0 - if {[catch { - foreach line [split \ - [exec nice -n 19 /mod/bin/stripts -q $file $tmp/shrunk] \ - "\n"] { - log $line 0 - } - } msg]} { - log "Error during shrink: $msg" 0 - system notify "$file - auto-shrink - error $msg." - system endop shrink - return - } - - # The following steps are structured to minimise the risk of - # things being left in an inconsistent state if the system goes - # into standby. Renames within the same filesystem are very - # quick so the risk is small, but even so... - - # Move the shrunken version back to the local directory. - foreach f [glob "$tmp/shrunk.*"] { - set ext [file extension $f] - file rename $f "${file}_shrunk${ext}" - # Set the file time to match the old file - file touch "${file}_shrunk${ext}" [$ts get file] - } - - # Remove the old recording (-> bin if undelete is installed) - safe_delete [$ts get file] "webif_autoshrink" - - # Finally, rename the shrunken recording again. - foreach ext $tsgroup { - set f "${file}_shrunk.$ext" - if {[file exists $f]} { - file rename $f "${file}.$ext" - } - } - $ts set_shrunk - log "Done... [endclock $size]" 0 - lappend processed_files [$ts get file] - runplugin postshrink $ts - system endop shrink -} - -proc do_decrypt {ts} { - global tmp dustbin processed_files - - set file [$ts get file] - set rfile [file rootname $file] - set bfile [file tail $file] - - if {![$ts flag "ODEncrypted"]} { - log " $file - Already decrypted." 2 - return - } - - if {[$ts flag "Encrypted"]} { - log " $file - Protected (Enc flag)." 2 - return - } - - if {![system dlnastatus]} { - log " $file - DLNA Server not running." 2 - set ::dlnaok 0 - return - } - - if {[inuse $ts]} { - log " $file - In use." - return - } - - # Check that the file is not already decrypted by analysing it. - set anencd [exec /mod/bin/stripts -qE $rfile] - if {$anencd != "1"} { - log " $file - already decrypted but the HMT flag is wrong." 0 - # Fix... - exec /mod/webif/lib/bin/fixencflags $file - return - } - - lassign [$ts dlnaloc "127.0.0.1"] url - if {$url ne ""} { - log " $file - has been indexed." - set helper 0 - } else { - log " $file - Not yet indexed, trying helper." - if {[catch { - lassign [system dlnahelper [file normalize $file]] url - } msg]} { - log " $file - $msg" - } - if {$url eq ""} { - log " $file - Can't use helper." - return - } - set helper 1 - } - - # Perform the decryption by requesting the file from the DLNA server. - oktorun - set size [$ts size] - dsc $size - runplugin predecrypt $ts - system startop decrypt $file - startclock - log " DECRYPT: $rfile" 0 - log " DLNA: $url" 0 - exec wget -O "$tmp/$bfile" $url - - # Release the helper lock once finished. - if {$helper} { system dlnahelper -release } - - if {[file size $file] != [file size "$tmp/$bfile"]} { - log " $file - File size mismatch." 0 - file tdelete "$tmp/$bfile" - system endop decrypt - return - } - - # Check if the file is in use. It is possible that the file is - # now being played even though it was free when decryption started. - if {[inuse $ts]} { - log " $file - In use." - file tdelete "$tmp/$bfile" - system endop decrypt - return - } - - # Copy the HMT file over for stripts - set thmt "$tmp/[file rootname $bfile].hmt" - file copy "$rfile.hmt" $thmt - # Check that the file is no longer encrypted by analysing it. - set anencd [exec /mod/bin/stripts -qE "$tmp/[file rootname $bfile]"] - file delete $thmt - - if {$anencd != "0"} { - log " $file - File did not decrypt properly." 0 - file tdelete "$tmp/$bfile" - if {[clock seconds] - [$ts get end] > 3 * 3600} { - system notify "$file - auto-decrypt failed." - } - system endop decrypt - return - } - - # Move the encrypted file out of the way. - file rename $file "$rfile.encrypted" - # Move the decrypted copy into place. - file rename "$tmp/$bfile" $file - # Set the file time to match the old file - file touch $file "$rfile.encrypted" - # Patch the HMT - quickest way to get back to a playable file. - exec /mod/bin/hmt -encrypted "$rfile.hmt" - - log " Removing/binning old copy." 0 - # Move the old recording to the bin if undelete is installed. - if {$dustbin ne ""} { - set bin [_del_bindir $file "webif_autodecrypt"] - set tail [file tail $rfile] - file rename "$rfile.encrypted" "$bin/$tail.ts" - foreach ext {nts hmt thm} { - if {[file exists "$rfile.$ext"]} { - file copy $rfile.$ext "$bin/$tail.$ext" - if {$ext eq "hmt"} { - # Patch the binned HMT back - exec /mod/bin/hmt +encrypted \ - "$bin/$tail.hmt" - } - } - } - } else { - file tdelete "$rfile.encrypted" - } - log "Done... [endclock $size]" 0 - lappend processed_files [$ts get file] - $ts unflag "ODEncrypted" - runplugin postdecrypt $ts - system endop decrypt -} - -proc do_mpg {ts} { - global tmp tsgroup processed_files - - set file [file rootname [$ts get file]] - - if {[file exists $file.mpg]} { - # Already done. - return - } - - if {[$ts flag "ODEncrypted"]} { - log " $file - Not decrypted." 2 - return - } - - if {[$ts get definition] eq "HD"} { - # Cannot extract a useful MPG from a HD recording. - return - } - - if {[inuse $ts]} { - log " $file - In use." - return - } - oktorun - system startop mpg [$ts get file] - runplugin prempg $ts - dsc [$ts size] - - log " MPG: $file" 0 - log " Converting..." 0 - if {[catch { - foreach line [split \ - [exec nice -n 19 /mod/bin/ffmpeg -y -benchmark -v 0 \ - -i $file.ts \ - -map 0:0 -map 0:1 \ - -vcodec copy -acodec copy $tmp/mpg.mpg] "\n"] { - log $line 0 - } - } msg]} { - log "Error during mpg extract: $msg" 0 - system notify "$file - auto-mpg - error $msg." - system endop mpg - return - } - - # Move the MPG into the local directory - file rename $tmp/mpg.mpg $file.mpg - lappend processed_files [$ts get file] - runplugin postmpg $ts - system endop mpg -} - -proc do_mp3 {ts} { - global tmp tsgroup processed_files - - set file [file rootname [$ts get file]] - - if {[file exists $file.mp3]} { - # Already done. - return - } - - if {[$ts flag "ODEncrypted"]} { - log " $file - Not decrypted." 2 - return - } - - if {[$ts get definition] eq "HD"} { - # Cannot extract a useful MP3 from a HD recording. - log " $file - High definition." 2 - return - } - - if {[inuse $ts]} { - log " $file - In use." - return - } - oktorun - system startop mp3 [$ts get file] - runplugin premp3 $ts - dsc [$ts size] - - log " MP3: $file" 0 - log " Converting... [$::settings audiomp3descr $::audiomp3]" 0 - if {[catch { - set cmd [list nice -n 19 \ - /mod/bin/ffmpeg -y -benchmark -v 0 -i "$file.ts" \ - -f mp3 -vn] - if {!$::audiomp3} { lappend cmd -acodec copy } - lappend cmd "$tmp/mp3.mp3" - foreach line [split [exec {*}$cmd] "\n"] { - log $line 0 - } - } msg]} { - log "Error during mp3 extract: $msg" 0 - system notify "$file - auto-mp3 - error $msg." - system endop mp3 - return - } - - if {[system pkginst id3v2]} { - log [exec /mod/bin/id3v2 \ - --song "[$ts get title]" \ - --comment "[$ts get synopsis]" \ - --album "[$ts get channel_name]" \ - --year "[clock format [$ts get start] -format {%Y}]" \ - "$tmp/mp3.mp3"] 0 - } - - # Move the MP3 into the local directory - file rename $tmp/mp3.mp3 $file.mp3 - lappend processed_files [$ts get file] - runplugin postmp3 $ts - system endop mp3 -} - -proc entries {dir callback {flag ""}} { - foreach entry [readdir -nocomplain $dir] { - if {$flag ne "" && ![file exists $flag]} { - log "Flag $flag disappeared mid-run, aborting." 0 - break - } - if {![string match {*.ts} $entry]} continue - if {[catch {set ts [ts fetch "$dir/$entry"]}]} continue - if {$ts == 0} continue - $callback $ts - } -} - -proc shrink {dir {flag ""}} { - log "SHRINK: \[$dir]" 2 - entries $dir do_shrink $flag -} - -proc decrypt {dir {flag ""}} { - log "DECRYPT: \[$dir]" 2 - if {$::dlnaok} { entries $dir do_decrypt $flag } -} - -proc mpg {dir {flag ""}} { - log "MPG: \[$dir]" 2 - entries $dir do_mpg $flag -} - -proc mp3 {dir {flag ""}} { - log "MP3: \[$dir]" 2 - entries $dir do_mp3 $flag -} - -proc expire {dir {flag ""}} { - log "EXPIRE: \[$dir]" 2 - - # type 0 keep {} days 2 keepnew 0 - set ax [dir expiry $dir] - log " $ax" 2 - if {![string is integer -strict $ax(keep)]} { set ax(keep) 0 } - if {![string is integer -strict $ax(days)]} { set ax(days) 0 } - - if {!$ax(keep) && !$ax(days)} { - log "Expire error ($dir): neither keep nor days set ($ax)" 0 - return - } - - if {$ax(type) eq "0"} { - set getter [lambda {ts} { list [$ts get end] }] - } else { - set getter [lambda {ts} { list [$ts lastmod] }] - } - - set entries [lsort \ - -command [lambda {a b} {upvar getter g; expr [$g $a] - [$g $b]}]\ - [lmap i [readdir -nocomplain $dir] { - if {![string match {*.ts} $i]} continue - if {[catch {set ts [ts fetch "$dir/$i"]}]} continue - if {$ts == 0} continue - list $ts - }]] - - # Entries are now sorted with oldest first, according to the selected - # criterion. - - set num [llength $entries] - log " Entries: $num" 2 - foreach e $entries { log " [file tail [$e get file]]" 2 } - - set now [clock seconds] - foreach ts $entries { - set file [$ts get file] - if {$num <= $ax(keep)} { - log " Remaining ($num) <= keep ($ax(keep))" 2 - break - } - log " [file tail $file]" - if {[inuse $ts]} { - log " In use, skipping." - continue - } - if {$ax(keepnew) ne "0" && [$ts flag New]} { - log " Unwatched, skipping." - continue - } - if {$ax(days) > 0} { - # Calculate age in days - set age $(($now - [$getter $ts]) / 86400.0) - log " $age days (cf. $ax(days))" - if {$age < $ax(days)} { - log " Too new, skipping." - continue - } - } - log " Removing." - runplugin preexpire $ts - if {![safe_delete $file]} { - log "Unknown error in safe_delete, aborting." 0 - break - } - log " EXPIRE: removed $file" 0 - runplugin postexpire $ts - incr ::recalc - incr num -1 - } -} - -proc scan_run {dir flag callback {seen {}}} { - global dustbin - - if {$dir eq $dustbin} return - - if {[specialdir $dir]} return - - file stat $dir st - set key "$st(dev):$st(ino)" - if {$key in $seen} { - log "Already seen $dir ($key)" 2 - return - } - lappend seen $key - - if {[file exists "$dir/.$flag"]} { $callback $dir } - - foreach entry [readdir -nocomplain $dir] { - if {[file isdirectory "$dir/$entry"]} { - scan_run "$dir/$entry" $flag $callback $seen - - file stat "$dir/$entry" st - set key "$st(dev):$st(ino)" - lappend seen $key - } - } -} - -proc xscan {dir attr {force 0} {recurse 1} {seen {}}} {{indent 0} {forceflag ""}} { - global dustbin - - incr indent 2 - - log "[string repeat " " $indent]\[$dir]" 2 - - if {$dir eq $dustbin} { - log "[string repeat " " $indent]Dustbin, skipping." 2 - incr indent -2 - return - } - - file stat "$dir/" st - - if {[specialdir $dir]} { - # Special folder - if {$st(dev) != $::rootdev} { - log "[string repeat " " $indent]Special folder on different device, skipping." 2 - incr indent -2 - return - } - if {$force} { - set force 0 - set forceflag "" - log "[string repeat " " $indent]Special folder, overriding recursion." 2 - } - } - - # Already seen - set key "$st(dev):$st(ino)" - if {$key in $seen} { - log "[string repeat " " $indent]Already seen, skipping." 2 - incr indent -2 - return - } - lappend seen $key - - # Recursion - if {!$force && [file exists "$dir/.auto${attr}r"]} { - log "[string repeat " " $indent] (R)" 2 - set force 1 - set forceflag "$dir/.auto${attr}r" - } - - oktorun - dsc - - if {$force} { - $attr $dir $forceflag - } elseif {[file exists "$dir/.auto$attr"]} { - $attr $dir "$dir/.auto$attr" - } - - foreach entry [readdir -nocomplain $dir] { - if {$recurse && [file isdirectory "$dir/$entry"]} { - xscan "$dir/$entry" $attr $force $recurse $seen - file stat "$dir/$entry" st - set key "$st(dev):$st(ino)" - lappend seen $key - } - } - - dorecalc $dir - - incr indent -2 -} - -proc scanup {dir flag} { - global root - - set rl [string length $root] - while {[string length $dir] >= $rl} { - if {[specialdir $dir]} { - return -1 - } - if {[file exists "$dir/.auto${flag}r"]} { - log "scanup: Found ${flag}r ($dir)" 2 - return 1 - } - set dir [file dirname $dir] - } - - return 0 -} - -proc scansingle {dirs} { - global modules root - - foreach dir $dirs { - log "Scanning single directory '$dir'" - foreach arg $modules { - set st [clock milliseconds] - set sup [scanup $dir $arg] - if {$sup == -1} { - log "Encountered special directory." - break - } - runplugin "pre${arg}singledir" $dir - xscan $dir $arg $sup 0 - runplugin "post${arg}singledir" $dir - log "$arg scan completed in [elapsed $st] seconds." - } - } -} - -proc process_queue {} { - global settings - log "Processing queue..." 2 - queue startup [$settings autokeep] - while {[llength [set q [queue pop]]]} { - oktorun - - if {[catch {set ts [ts fetch [$q get file]]}]} { - $q update FAILED "Could not open recording." 1 - continue - } - if {$ts == 0} { - $q update FAILED "Could not open recording." 1 - continue - } - set act [$q get action] - if {![dict exists $::queue_actions $act]} { - log "Unknown queue action, $act" - $q update FAILED "Unknown queue action, $act" 1 - continue - } - set func $::queue_actions($act) - $q update RUNNING "Started at [clock format [clock seconds]]" - set start [clock seconds] - if {[catch {$func $ts} msg]} { - $q update FAILED "$msg" 1 - continue - } - set elapsed $([clock seconds] - $start) - $q update COMPLETE \ - "Completed at [clock format [clock seconds]]" 0 $elapsed - } -} - -file stat "$root/" rootstat -set rootdev $rootstat(dev) -log "Root device: $rootdev" 2 - -###################################################################### - -if {[lindex $argv 0] eq "-singledir"} { - scansingle [lrange $argv 1 end] -} elseif {[lindex $argv 0] eq "-queue"} { - process_queue -} elseif {[llength $argv] > 0} { - set loglevel 2 - foreach arg $argv { - set st [clock milliseconds] - if {$arg ni $modules} { - log "Unknown scan type: $arg" - continue - } - log "$arg scan starting." - set processed_files {} - runplugin "pre${arg}scan" - xscan $root $arg - runplugin "post${arg}scan" processed_files - log "$arg scan completed in [elapsed $st] seconds." - } -} else { - process_queue - foreach arg $modules { - set st [clock milliseconds] - log "$arg scan starting." - set processed_files {} - runplugin "pre${arg}scan" - xscan $root $arg - runplugin "post${arg}scan" processed_files - log "$arg scan completed in [elapsed $st] seconds." - } - $settings autolast [clock seconds] -} - -if {!$prelocked} { release_lock webif_auto } - -log "Auto processing completed in [elapsed $scanstart] seconds." - +### +###if {[lindex $argv 0] eq "-singledir"} { +### scansingle [lrange $argv 1 end] +###} elseif {[lindex $argv 0] eq "-queue"} { +### process_queue +###} elseif {[llength $argv] > 0} { +### set loglevel 2 +### foreach arg $argv { +### set st [clock milliseconds] +### if {$arg ni $modules} { +### log "Unknown scan type: $arg" +### continue +### } +### log "$arg scan starting." +### set processed_files {} +### runplugin "pre${arg}scan" +### xscan $root $arg +### runplugin "post${arg}scan" processed_files +### log "$arg scan completed in [elapsed $st] seconds." +### } +###} else { +### process_queue +### foreach arg $modules { +### set st [clock milliseconds] +### log "$arg scan starting." +### set processed_files {} +### runplugin "pre${arg}scan" +### xscan $root $arg +### runplugin "post${arg}scan" processed_files +### log "$arg scan completed in [elapsed $st] seconds." +### } +### $settings autolast [clock seconds] +###} +### +###if {!$prelocked} { release_lock webif_auto } +### +###log "Auto processing completed in [elapsed $scanstart] seconds." +### diff --git a/webif/lib/plugin b/webif/lib/plugin index 4fed142b..8f5f3082 100644 --- a/webif/lib/plugin +++ b/webif/lib/plugin @@ -1,6 +1,7 @@ -proc eval_plugins {hook {verbose 0} {callback ""}} { - foreach plugin [lsort [glob -nocomplain /mod/webif/plugin/*]] { +proc eval_plugins {hook {verbose 0} {callback ""} {dir /mod/webif/plugin}} { + foreach plugin [lsort [glob -nocomplain "$dir/*"]] { + if {![file isdirectory $plugin]} continue if {[file exists "$plugin/.disabled"]} continue if {[file isfile "$plugin/$hook.hook"]} { if {$callback ne ""} { diff --git a/webif/lib/queue.class b/webif/lib/queue.class index 9573db98..88299858 100644 --- a/webif/lib/queue.class +++ b/webif/lib/queue.class @@ -3,16 +3,25 @@ if {![exists -proc class]} { package require oo } if {![exists -proc sqlite3.open]} { package require sqlite3 } class queue { - id -1 - dat 0 - file "" - action "" - log "" - status "" - retry 0 - elapsed 0 + id -1 + file "" + action "" + start 0 + status "" + log "" + runtime 0 + retries 0 + submitted 0 } +# +# Queue status values: +# PENDING +# FAILED +# INTERRUPTED +# COMPLETE +# DEFER + proc {queue dbhandle} {args} { if {"-close" in $args} { if {[info exists ::queue::db]} { @@ -32,15 +41,25 @@ proc {queue dbhandle} {args} { $::queue::db query { create table queue( id integer primary key autoincrement, - dat integer, file text, action text, + start integer default 0, status text default 'PENDING', log text default '', - elapsed integer, - retry integer default 0 + runtime integer, + retries integer default 0, + submitted integer ); } + $::queue::db query { + create table config( + var text, + val text + ); + } + $::queue::db query { + insert into config values('version', 2); + } $::queue::db query { create unique index file on queue(file, action); } @@ -51,23 +70,6 @@ proc {queue dbhandle} {args} { return $::queue::db } -queue method update {_status {_log ""} {_retry 0} {_elapsed 0}} { - set db [queue dbhandle] - $db query { - update queue - set status = '%s', - log = '%s', - retry = retry + %s, - elapsed = %s - where id = %s - } $_status $_log $_retry $_elapsed $id - - set status $_status - set log $_log - incr retry $_retry - set elapsed $_elapsed -} - proc {queue startup} {{days 7}} { if {$days == 0} { set days 7 } set db [queue dbhandle] @@ -75,33 +77,50 @@ proc {queue startup} {{days 7}} { update queue set status = 'INTERRUPTED', log = 'Job will be retried automatically.', - retry = retry + 1 + retries = retries + 1 where status in ('RUNNING', 'INTERRUPTED') - and retry < 5 + and retries < 5 } $db query { update queue set status = 'FAILED', - log = 'Too many retries.', - retry = retry + 1 + log = 'Too many retries.' where status in ('RUNNING', 'INTERRUPTED') + and retries >= 5 + } + $db query { + update queue + set status = 'PENDING' + where status = 'DEFER' } $db query { delete from queue where status in ('COMPLETE', 'FAILED') - and dat < %s + and submitted < %s } [expr [clock seconds] - 86400 * $days] } +proc {queue fetch} {id} { + set db [queue dbhandle] + + foreach row [$db query { + select * from queue + where id = %s + } $id] { + return [queue new $row] + } + return {} +} + proc {queue insert} {ts action} { set db [queue dbhandle] $db query { - insert or ignore into queue(dat, file, action) + insert or ignore into queue(submitted, file, action) values(%s, '%s', '%s') } [clock seconds] [file normalize [$ts get file]] $action - return [$db lastid] + return [queue fetch [$db lastid]] } proc {queue delete} {ts {action "*"}} { @@ -136,7 +155,7 @@ proc {queue resubmit} {id} { set q " update queue - set status = 'PENDING' + set status = 'PENDING', retries = 0 where id = '%s' and status in ('FAILED') " @@ -188,38 +207,42 @@ proc {queue pending} {} { foreach row [$db query { select * from queue where status in ('PENDING', 'INTERRUPTED') - order by id - }] { + and start < %s + order by id desc + } [clock seconds]] { lappend ret [queue new $row] } return $ret } -proc {queue pop} {} { - set db [queue dbhandle] - - foreach row [$db query { - select * from queue - where status in ('PENDING', 'INTERRUPTED') - order by id - limit 1 - }] { - return [queue new $row] - } - return {} -} - proc {queue size} {} { - set db [queue dbhandle] - - set num 0 - set ret [$db query { - select count(*) from queue - where status in ('PENDING', 'INTERRUPTED') - }] - if {[llength $ret]} { - lassign [lindex $ret 0] x num - } - return $num + return [llength [queue runcandidates]] +} + +queue method update {_status {_log ""} {_retries 0} {_runtime 0}} { + set db [queue dbhandle] + $db query { + update queue + set status = '%s', + log = '%s', + retries = retries + %s, + runtime = %s + where id = %s + } $_status $_log $_retries $_runtime $id + + set status $_status + set log $_log + incr retries $_retries + set runtime $_runtime +} + +queue method set {var val} { + set db [queue dbhandle] + $db query { + update queue + set %s = '%s' + where id = %s + } $var $val $id + set $var $val }