first commit

This commit is contained in:
Bob Buxton 2020-10-21 17:37:23 +01:00
commit fb328daf43
22 changed files with 837 additions and 0 deletions

10
CONTROL/control Normal file
View File

@ -0,0 +1,10 @@
Package: qtube
Priority: optional
Section: misc
Version: 0.1.0-2
Architecture: mipsel
Maintainer: mymsman
Depends: webif(>=1.4.8), youtube-dl (>=2020.03.24)
Description: Simple webif to queue youtube downloads to run in the Auto process queue
Tags: http://wiki.hummy.tv/wiki/qtube

56
bin/qtube Executable file
View File

@ -0,0 +1,56 @@
#!/mod/bin/jimsh
source /mod/webif/lib/setup
require lock system.class ts.class queue.class \
settings.class plugin
set loglevel 2
set logfd stdout
proc log {msg {level 1}} {
if {$level > $::loglevel} return
puts $::logfd "[\
clock format [clock seconds] -format "%d/%m/%Y %H:%M:%S"\
] QT([pid])- $msg"
flush $::logfd
}
set ct [llength $argv]
set optct $($ct-2)
set url [lindex $argv $($ct -1)]
set optlist [lrange $argv 0 $optct]
set strtk [lsearch $optlist "-start"]
set start [clock seconds]
if {$strtk!=-1} {
# -start specified
set time [lindex $argv $($strtk +1)]
set optlist [lreplace $optlist $strtk $($strtk +1)]
if {[string is integer -strict $time]} {
set start $time
} else {
catch {
set tstart [clock scan $time -format %H:%M]
if {$tstart<$start} {incr tstart 86400}
set start $tstart
}
}
if {$start == 0} {set start [clock seconds]}
}
if {$optlist == "{}"} {set optlist ""}
log "Starting URL: $url options: $optlist" 0
# queue with system Q
set q [{queue insert} -hold $url qtube]
$q set start $start
$q set args $optlist
set Qid [$q get id]
$q submit
log "Queued id: $Qid URL: $url options: $optlist" 0

12
webif/plugin/qtube/dequeue.jim Executable file
View File

@ -0,0 +1,12 @@
#!/mod/bin/jimsh
# DeQueue url from backround queue
package require cgi queue class
source /mod/webif/lib/setup
httpheader
puts "DeQueue starting"
set url [cgi_get url -]
::detectads::delete_entry_queue_db $ts

View File

@ -0,0 +1,3 @@
register_module "qtube" "/plugin/qtube/"
#system plog qtube "gohook mod $module last $lastmodule"

Binary file not shown.

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.6 KiB

70
webif/plugin/qtube/index.jim Executable file
View File

@ -0,0 +1,70 @@
#!/mod/bin/jimsh
package require cgi
source /mod/webif/lib/setup
require ts.class pretty_size queue.class
source /mod/webif/plugin/qtube/timelist.jim
jqplugin enadis iajax
jscss script.js style.css
header
# escape single quotes in the urlname and dirname so they don't mess up the javascript
#set non_escaped_url $rurl
#set dir [string map {' \\'} [url dirname $rurl]]
#set rurl [string map {' \\'} $rurl]
puts "
<fieldset class=cleft>
<legend><a href=\"https://wiki.hummy.tv/wiki/Custom_Firmware_Package_Notes#Youtube-dl\" target=\"_blank\"><img src=\"/plugin/qtube/img/qtube.png\" style=\"vertical-align: middle\" width=64px alt=\"qtube icon\"/> YouTube (& BBC iPlayer) download</a></legend>
<p>See <a href=\"https://wiki.hummy.tv/wiki/Custom_Firmware_Package_Notes#Youtube-dl\" target=\"_blank\">Wiki page</a> for a brief description of available options and
and <a href=\"https://github.com/rg3/youtube-dl/blob/master/README.md#readme\" target=\"_blank\">Github Notes</a> for Full reference notes
<p>
Youtube-dl has a config file at : <a href=\"/edit/edit.jim?file=/mod/etc/youtube-dl.conf&backdesc=Back&backlink=/plugin/qtube\">/mod/etc/youtube-dl.conf</a>
<table class=keyval cellpadding=5>
<tr><th>Url:</th> <td>
<input id=url
class=\"ui-widget-content ui-corner-all\"
autocomplete=yes
type=text size=150>
</td></tr>
"
puts "
</td></tr>
<tr><th>Process options</th><td>
<input id=opts
class=\"ui-widget-content ui-corner-all\"
autocomplete=yes
type=text size=150>
</td></tr>
</table>
"
puts "
<div id=buttons>
<button id=run>Run Download now</button>
<button id=queue>Run Download in background</button>
<select id=queue_time name=queue_time
class=\"ui-widget-content ui-corner-all\" >
"
timelist 0
puts "
</select>
<button id=viewqueue><img src=/img/queueps.png class=qb> View Queue</button>
<button id=viewlog>View qtube.log</button>
</div>
"
puts "
<div id=results class=\"hidden\">
<div class=\"running blood\">
<img src=/img/loading.gif>
Analysing recording, please wait...
</div>
<div id=results_inner></div>
<div class=\"running blood\">
<img src=/img/loading.gif> Still working, please be patient...
</div>
"

View File

@ -0,0 +1,7 @@
menuitem \
"Queue video download" \
"/plugin/qtube/img/qtube.png" \
"/go/qtube"
# "/plugin/qtube/index.jim"

346
webif/plugin/qtube/qtube.jim Executable file
View File

@ -0,0 +1,346 @@
proc ::chasedecrypt::delete_orphans {} {
if {![catch {exec /mod/bin/pgrep -x chaseget }]} {return}
set fl [glob -nocomplain "/mod/tmp/*-inp.ts" "/mod/tmp/*-dec.ts"]
log "orphan file list $fl" 2
foreach file $fl {
if {![system inuse $file]} {
set bname [file rootname $file]
file delete -force $bname.hmt
file delete -force $bname.nts
file delete -force $file
log "Deleted orphan file $file"
}
}
}
# Parse command options and apply defaults
proc ::chasedecrypt::checkopts {argv} {
set ::optlist ""
set ::opt "-h"
set ::debug 0
set parmerror 0
set settings [settings]
set ::autologlevel [$settings _nval_setting "autolog"]
# List of options with default values
set optarray {
delorig 0
misrdsec 0
delsec 210 }
# Override default from settings DB
foreach {key defvalue} [array get optarray] {
set ::opts($key) [$settings _nval_setting "chasedecrypt_$key"]
if {$::opts($key)==0} {set ::opts($key) $defvalue}
}
# Handle text setting for oher options
set otheropts [$settings _tval_setting "chasedecrypt_otheropts"]
# Parse argument lists
foreach argl [list $otheropts $argv] {
#set ::optlist ""
log "arg list $argl" 2
for {set ix 0} {$ix < [llength $argl]} {incr ix} {
set arg [lindex $argl $ix]
#check if option in optarray list
if {[string range $arg 0 0] == "-"} {
set argx [string tolower [string range $arg 1 end]]
if {[dict exists $optarray $argx]} {
incr ix
set val [lindex $argl $ix]
set nval $val
if {$val eq "y"} {set nval 1}
if {$val eq "n"} {set nval 0}
if {![string is double -strict $nval]} {
log "Option $arg value ($val) is not y, n or numeric" 0
incr ix -1
set parmerror 1
continue
}
lappend ::optlist $arg
lappend ::optlist $val
set ::opts($argx) $nval
#puts "found -$argx value $val"
continue
}
}
# check other options
switch -- $arg {
-debug -
-d {
set ::debug 1
set ::loglevel 2
set ::auto::loglevel 2
lappend ::optlist $arg
}
default {
log "Unrecognized option: $arg" 0
set parmerror 1
continue
}
}
}
}
if {$parmerror} {
log "Parameter errors found"
exit
}
}
proc ::chasedecrypt::chancheck {ts} {
# Check recording channel against inclusion list prior to queueing for analysis
set file [$ts get file]
set channel [$ts get channel_name]
log " ChaseDecrypt: Checking $file ($channel) for inclusion" 0
set dir [file dirname $file]
#if {[file exists "$dir/.autonochasedecrypt"]} {
# log " ChaseDecrypt: No Ad-detection folder flag set $file" 0
# return 0
#}
if {![$ts flag "ODEncrypted"]} {
log " ChaseDecrypt: Already decrypted $file" 0
return 0
}
# ignore FlatView directory
set settings [settings]
set fvdir [$settings _tval_setting "fv_dir"]
if {[file tail $dir] == $fvdir} {
log " ChaseDecrypt: Flat view directory, skipping $file" 0
return 0
}
set decOK 0
set opts ""
if {![file exists "$dir/.chasedecrypt"]} {
# open and read configuration file
set cf "/mod/etc/chasedecrypt.conf"
if {![file exists $cf]} {
file copy /mod/webif/plugin/chasedecrypt/default.conf $cf
}
if {![catch {set fp [open $cf r]}]} {
set clist [split [read $fp] "\n"]
} else {
set clist {}
}
# Match channel against exclusion list
foreach chan $clist {
if {![string length $chan]} continue
log "Checking exclusion entry: $chan" 2
if {[string match -nocase $chan $channel]} {
log "Matched inclusion entry $chan" 0
set decOK 1
break
}
}
} else {
# Read options from .chasedecrypt file
set decOK 1
set ado_file "$dir/.chasedecrypt"
set hand [open $ado_file]
set opts [gets $hand]
close $hand
}
::chasedecrypt::checkopts $opts
if {$decOK} {
::chasedecrypt::chaserun $ts
} else {
log " ChaseDecrypt: $file not eligible for chase decryption" 0
}
}
proc ::chasedecrypt::chaserun {ts} {
# run decryption against currently recording program using dlna helper
set file [$ts get file]
log "==ChaseDecrypt Chase Run: $file $::optlist" 0
set retcode "OK"
set retmsg "Unknown = check log"
set qtime 0
set warning ""
set cropcmd " "
# Check for and delete any oprhaned files
::chasedecrypt::delete_orphans
set statustok [system startop -multiple chasedecrypt $file]
set size [$ts size]
set numAdBreaks 0
# set each option from settings/overrides
foreach {key value} [array get ::opts] {
set $key $value
}
set stime [$ts get start]
set etime [$ts get end]
set ctime [clock seconds]
# Check file sharing enabled
if {[system param DMS_START_ON]} {
log "Content Sharing Enabled" 2
} else {
puts "Content Sharing Disabled -cannot decrypt files"
log "Content Sharing Disabled" 0
return {"FAILED" "Content Sharing Disabled"}
}
set bname [file rootname [file tail $file]]
set iname "$bname-inp"
set tname "$bname-dec"
set bpath [file dirname $file]
set tpath $bpath
set ipath "/mod/tmp"
#set tpath "/mod/tmp"
set bfile "$file"
set ifile "$ipath/$iname.ts"
set tfile "$tpath/$tname.ts"
set status [$ts get status]
# Open recording to lock against Auto and Flatten
set recording [open $file r]
if {($stime +$delsec) > $ctime} {
log "Waiting for recording $delsec seconds" 1
sleep $($stime +$delsec- $ctime)
set ctime [clock seconds]
}
if {![acquire_lock $file]} {
log "Cannot acquire exclusive lock $file, terminating." 0
return {"DEFER" "Cannot acquire exclusive lock"}
}
set start [clock milliseconds]
log "starting" 2
# Create links to input in tmp for retrieval
file delete -force "$ipath/$iname.ts"
catch {file link -hard "$ipath/$iname.ts" "[file normalize [file rootname $file].ts]"}
file delete -force "$ipath/$iname.nts"
catch {file link -hard "$ipath/$iname.nts" "[file normalize [file rootname $file].nts]"}
file delete -force "$ipath/$iname.hmt"
catch {file link -hard "$ipath/$iname.hmt" "[file normalize [file rootname $file].hmt]"}
set its [ts fetch $ifile]
# Use link for .nts so updates are visble
file delete -force "$tpath/$tname.nts"
catch {file link -hard "$tpath/$tname.nts" "[file normalize [file rootname $file].nts]"}
# Copy sidecar files and update program title, encryption flag
file copy -force "[file rootname $file].hmt" "$tpath/$tname.hmt"
# Update output hmt to valid, decrypted, ad-detected empty file
set title [$ts get title]
set guidance [$ts get guidance]
exec hmt "-encrypted" "$tpath/$tname.hmt"
exec hmt "-protect" "$tpath/$tname.hmt"
exec hmt "+patch8=0x28c:2" "$tpath/$tname.hmt"
set ftime [clock format $stime -format "%Y%m%d%H%M.%S"]
exec touch $tfile -t $ftime
set tts [ts fetch $tfile]
#set tfilesize [file size $tfile] ;# Restart point
set tfilesize 0 ;# Silence/nsplice cant handle restart midway so force total file retrieval
set newtitle [concat $title "-Decrypt"]
exec hmt "+settitle=${newtitle}" "$tpath/$tname.hmt"
exec /mod/bin/chaseget $ifile $tfilesize > $tfile
# Delete links to input from /mod/tmp
$its delete
# Check that complete file has been retrieved & detected
set ts [ts fetch $file]
set tfilesize [file size $tfile]
set filesize [file size $file]
set missing $($filesize -$tfilesize)
set lengtherr 0
set stime [$ts get start]
set etime [$ts get end]
set durn $($etime-$stime)
set misstime $(round(abs($missing)/($filesize/$durn)))
# Recopy hmt to get completion details
file copy -force "[file rootname $file].hmt" "$tpath/$tname.hmt"
# Update output hmt to valid, decrypted, nonprotected file
exec hmt "-encrypted" "$tpath/$tname.hmt"
exec hmt "-protect" "$tpath/$tname.hmt"
exec hmt "+settitle=${newtitle}" "$tpath/$tname.hmt"
set newtitle $title
if {$missing != 0} {
# Length error in decryption
set lengtherr 1
set warning "Incomplete data retrieval $missing bytes missing ([clock format $misstime -format %T])"
log "$file $warning" 0
set newtitle "$title -Len err ([clock format $misstime -format %T])"
exec hmt "+settitle=$newtitle" "$tpath/$tname.hmt"
if {$misstime >= $misrdsec} {
# system notify "ChaseDecrypt $file $warning"
} else {
# ignore the problem
set lengtherr 0
}
}
if {$lengtherr == 0} {
# All OK (as far as we can tell)
close $recording
if {$delorig && ![system inuse $file]} { # Delete original file no longer wanted
if {$delorig == 2} {
set del [$ts delete]
log "file $file deleted, code $del" 1
} else {
set del [safe_delete $file chasedecrypt]
log "$file dustbinned, code $del" 2
}
if {$del && ![system inuse $tfile]} { # Rename decrypt file to original if not in use
exec hmt "+settitle=$newtitle" "$tpath/$tname.hmt"
ts renamegroup "$tpath/$tname.ts" "$bname"
log "$tname renamed $bname" 1
}
} else { # Keeping original, dont rename
}
} else {
# Somethings amiss with file length, keep original and delete -dec
if {![system inuse $tfile]} {
[safe_delete $tfile chasedecrypt]
log "$tfile dustbinned" 2
}
close $recording
}
#set elapsedSeconds [elapsed $start]
#set elapsedTime [clock format $(round($elapsedSeconds)) -format "%H:%M:%S"]
#log "done...processed $file in ${elapsedSeconds}s $elapsedTime " 0
log "done...processed $file" 0
system endop $statustok
release_lock $file
log "=============================================================" 1
return
}

View File

@ -0,0 +1,51 @@
#source /mod/webif/plugin/qtube/qtube.jim
alias ::qtube::log ::auto::log
alias ::qtube::elapsed ::auto::elapsed
set ::logfd $::auto::logfd
proc qlog {msg {level 1}} {
puts $::qlogfd "[\
clock format [clock seconds] -format "%d/%m/%Y %H:%M:%S"\
] - $msg"
flush $::qlogfd
}
proc ::qtube::dequeue {q url} {
set retcode {"FAILED" "Unknown = check qtube.log"}
set opts [$q get args]
set ::Qid [$q get id]
set logfile "/mod/tmp/qtube.log"
set ::qlogfd [open $logfile "a+"]
qlog "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" 0
log "Starting queued download URL $url Options $opts QID $::Qid" 0
qlog "Starting queued download URL $url Options $opts QID $::Qid" 0
log "See $logfile for details" 0
# Fix to cmd invocation thanks to @/df
# https://hummy.tv/forum/threads/qtube-webif-front-end-for-youtube-dl.8948/page-7#post-139719
set cmd {youtube --newline}
lappend cmd {*}$opts $url
if {[catch {exec {*}[lmap e $cmd {concat $e}] 2>@$::qlogfd \
| awk {{print strftime("%d/%m/%Y %H:%M:%S -"), $0; fflush(); }} \
>@$::qlogfd } msg catchopts]
} {
log "Caught error: $msg" 0
log "$catchopts" 2
qlog "Caught error: $msg" 0
qlog "$catchopts" 2
set retcode [list "FAILED" "$msg - check qtube.log"]
} else {
log "Completed queued download of $url Options $opts" 0
qlog "Completed queued download of $url Options $opts" 0
set retcode {"OK" "Check qtube.log"}
}
close $::qlogfd
return $retcode
}
::auto::register qtube 200

21
webif/plugin/qtube/queue.jim Executable file
View File

@ -0,0 +1,21 @@
#!/mod/bin/jimsh
# Queue url for backround download
package require cgi
source /mod/webif/lib/setup
#roc log {msg {level 1}} {
# puts "[\
# clock format [clock seconds] -format "%T"\
# ] - $msg"
# flush stdout
#
httpheader
set url [cgi_get url -]
set queue_time [cgi_get queue_time 0]
set opts [cgi_get opts '']
#puts "queuing $url $opts"
exec {*}qtube -start $queue_time {*}$opts $url >@stdout

48
webif/plugin/qtube/runnow.jim Executable file
View File

@ -0,0 +1,48 @@
#!/mod/bin/jimsh
# Queue url for backround download
package require cgi
source /mod/webif/lib/setup
proc log {msg {level 1}} {
puts "[\
clock format [clock seconds] -format "%T"\
] - $msg"
flush stdout
puts $::qlogfd "[\
clock format [clock seconds] -format "%T"\
] - $msg"
flush $::qlogfd
}
httpheader
set logfile "/mod/tmp/qtube.log"
set ::qlogfd [open $logfile "a+"]
set url [cgi_get url "http://example.com"]
set opts [cgi_get opts ""]
set sess [clock format [clock seconds] -format "qtube%H%M"]
log "--------------------------------------------------------------"
log "Starting immediate download of $url Options $opts"
log "Be VERY patient - it can take a couple of minutes for download to start!"
# Fix to cmd invocation thanks to @/df
# https://hummy.tv/forum/threads/qtube-webif-front-end-for-youtube-dl.8948/page-7#post-139719
#set cmd [list "abduco" "-n" $sess "youtube" "--newline"]
set cmd [list "youtube" "--newline"]
lappend cmd {*}$opts $url
log $cmd
log "{*}[lmap e $cmd {concat $e}]"
if {[catch {exec {*}[lmap e $cmd {concat $e}] \
| awk {{print strftime("%H:%M:%S -"), $0; fflush(); }} \
| tee -i -a $logfile \
>@stdout 2>@1} msg catchopts]
} {
log "Caught error: $msg"
#log "$catchopts"
} else {
log "Completed immediate download of $url Options $opts"
}
close $::qlogfd

18
webif/plugin/qtube/save.jim Executable file
View File

@ -0,0 +1,18 @@
#!/mod/bin/jimsh
package require cgi
source /mod/webif/lib/setup
require settings.class
httpheader
set delorig [cgi_get chasedecrypt_delorig 0]
set otheropts [cgi_get chasedecrypt_otheropts ""]
[settings new] _nval_setting "chasedecrypt_delorig" $delorig
[settings new] _tval_setting "chasedecrypt_otheropts" $otheropts
puts "Settings saved."

View File

@ -0,0 +1,101 @@
var curval = 0;
$(function() {
$('#back').button({
icons : {
primary : "ui-icon-arrowreturnthick-1-w"
}
}).on('click', function() {
window,history.back();
});
function loaddata(data, isfinal) {
if (isfinal)
$('#results_inner').text(data);
else
$('#results_inner').append(data);
}
$('#run').button({
icons : {
primary : "ui-icon-play"
}
}).on('click', function() {
$('#results').removeClass('hidden').show();
$('.running').slideDown('slow');
$('#results_inner').empty();
$.ajax({
type : "GET",
url : 'runnow.jim',
data : {
'url' : $('#url').val() ,
'opts' : $('#opts').val()
},
progress : loaddata,
progressInterval : 5000,
timeout : 86400000,
success : function(data) {
$('.running').slideUp('slow');
loaddata(data, true);
loaddata('\n\n=== Done ===');
},
error : function(_, _, e) {
if (window.console)
console.log("ajax error");
$('.running').slideUp('slow');
alert(e);
}
});
});
$('#queue').button({
icons : {
primary : "ui-icon-play"
}
}).on('click', function() {
$('#results').removeClass('hidden').show();
$('.running').slideDown('slow');
$('#results_inner').empty();
$.ajax({
type : "GET",
url : 'queue.jim',
data : {
'url' : $('#url').val() ,
'queue_time': $('#queue_time').val(),
'opts' : $('#opts').val()
},
progress : loaddata,
progressInterval : 1000,
timeout : 50000,
success : function(data) {
$('.running').slideUp('slow');
loaddata(data, true);
loaddata('\n\n=== Check qtube.log later to see download progress ===');
},
error : function(_, _, e) {
if (window.console)
console.log("ajax error");
$('.running').slideUp('slow');
alert(e);
}
});
});
$('#viewqueue').button().click(function() {
window.location = '/diag/queue/';
});
$('#viewlog').button().click(function() {
window.location = '/log/?log=/mod/tmp/qtube.log';
});
});

View File

@ -0,0 +1,38 @@
$(function() {
// cloned from channeldel package - mymsman 150331
$('#back').button({
icons : {
primary : "ui-icon-arrowreturnthick-1-w"
}
}).on('click', function() {
window.location = '/settings/settings.jim'
});
$('.move').button().click(function(e) {
e.preventDefault();
$('#chanlist' + ' option:selected').each(function() {
var box = $('#exclist');
box.val(box.val() + $.trim($(this).val()) + '\n');
$(this).remove();
});
});
$('#excsave').ajaxForm({
target: '#saveresults',
success: function() {
$('#saveresults')
.css('font-style', 'italic')
.show('slow')
.delay(2000)
.fadeOut('slow');
}
});
$('#save').button();
});

View File

@ -0,0 +1,7 @@
register_statusop chasedecrypt "Chase Decrypting-" "/plugin/chasedecrypt/img/chasedecrypt.png"
#Based on
# Icon by Ivan Boyko https://www.iconfinder.com/icons/310307/running_sport_icon
# Creative Commons (Attribution 3.0 Unported) https://creativecommons.org/licenses/by/3.0/

View File

@ -0,0 +1,27 @@
#buttons
{
margin-top: 5px;
}
#results
{
margin-top: 1em;
}
#results_inner
{
margin-top: 1em;
overflow: auto;
white-space: pre;
}
img.qb
{
height: 14px;
position: relative;
top: 3px;
}

19
webif/plugin/qtube/timelist.jim Executable file
View File

@ -0,0 +1,19 @@
# Build HTML timeselector option list
proc timelist {sel} {
puts -nonewline "<option value=\"0\""
if {$sel eq 0} {
puts -nonewline " selected=yes"
}
puts ">ASAP</option>"
for {set t 0} {$t <1440} {incr t 30} {
# (mis)use the the fact of 60 sec per min 60 min per hour to format hh:mm, t is min per period
set ft [clock format $t -format %M:%S]
puts -nonewline "<option value=\"$ft\""
if {$sel eq $ft} {
puts -nonewline " selected=yes"
}
puts ">$ft</option>"
}
}

View File

@ -0,0 +1,3 @@
tb "/plugin/qtube/img/qtube.png" "Q Download" "/go/qtube"