tvdb integration

git-svn-id: file:///root/webif/svn/pkg/webif/trunk@2382 2a923420-c742-0410-a762-8d5b09965624
This commit is contained in:
hummypkg 2015-03-09 22:19:12 +00:00
parent 9b9032eee8
commit bf1bd66bbe
17 changed files with 647 additions and 12 deletions

View File

@ -1,10 +1,10 @@
Package: webif
Priority: optional
Section: web
Version: 1.2.2-1
Version: 1.2.2-3
Architecture: mipsel
Maintainer: af123@hummypkg.org.uk
Depends: webif-channelicons(>=1.1.16),lighttpd(>=1.4.35-2),jim(>=0.76),jim-oo,jim-sqlite3(>=0.76),jim-cgi(>=0.7),jim-binary(>=0.76),service-control(>=2.1),busybox(>=1.20.2-1),lsof(>=4.87),epg(>=1.2.0),hmt(>=2.0.3),ssmtp,anacron,trm(>=1.1),openssl-command,nicesplice,id3v2,file,rsvsync(>=1.0.2),webif-charts(>=1.2-1),stripts(>=1.2.5-3),smartmontools,tmenu(>=1.08),ffmpeg,id3v2,multienv(>=1.6),tcpping(>=1.1),e2fsprogs,wireless-tools(>=29-1),dbupdate,mongoose,recmon(>=2.0.2)
Depends: webif-channelicons(>=1.1.17),lighttpd(>=1.4.35-2),jim(>=0.76),jim-oo,jim-sqlite3(>=0.76),jim-cgi(>=0.7),jim-binary(>=0.76),service-control(>=2.1),busybox(>=1.20.2-1),lsof(>=4.87),epg(>=1.2.0),hmt(>=2.0.3),ssmtp,anacron,trm(>=1.1),openssl-command,nicesplice,id3v2,file,rsvsync(>=1.0.2),webif-charts(>=1.2-1),stripts(>=1.2.5-3),smartmontools,tmenu(>=1.08),ffmpeg,id3v2,multienv(>=1.6),tcpping(>=1.1),e2fsprogs,wireless-tools(>=29-1),dbupdate,mongoose,recmon(>=2.0.2)
Suggests:
Description: An evolving web interface for the Humax.
Tags: http://hummy.tv/forum/threads/5866/

View File

@ -361,7 +361,9 @@ puts {
</form>
</div>
<div id=dialogue></div>
<div id=dialogue>
<img src=/img/loading.gif>Retrieving data...
</div>
<div id=confirm title="Confirmation Required"></div>
<div id=pwdialogue style="display: none">
<center>

View File

@ -32,7 +32,33 @@ if {$type eq "ts"} {
<th>Synopsis</th>
<td>[$ts get synopsis]</td>
</tr><tr>
<th>Episode</th>
"
# Causes other series information to be automatically populated
set epname [$ts episode_name]
set series [$ts series_name]
set sdata [$ts get tvdb_series]
set idata [$ts get tvdb_data]
if {[llength $sdata]} {
set sid [$sdata get seriesid]
if {$sid} {
append series "/<a href=/db/index.jim?db=$sid.db>$sid</a>"
}
}
puts "<td>[$ts epstr] - $epname (<i>$series</i>)<td>
</tr><tr>
"
if {[llength $sdata] && [llength $idata]} {
puts "</tr><tr><th class=blood>DEBUG</th><td class=footnote>
Found using [$ts get tvdb_method]
<br>
$idata(overview)
<br>
[$sdata get _phrases]
<br>
[lrange [$sdata get _smatches] 0 8]
</td></tr>"
}
if {[$ts flag "Guidance"] > 0} {
puts "

View File

@ -769,7 +769,7 @@ var dmenuclick = function(action, el, pos)
draggable: true, resizable: true,
buttons: $buttons,
close: function(e,u) { $('#dialogue').empty().html(
'<img src="/img/loading.gif" alt="loading">'); }
'<img src="/img/loading.gif">Retrieving data...'); }
});
function doplay()

View File

@ -11,6 +11,8 @@ jscss script.js
set databases [glob /var/lib/humaxtv/*.db]
lappend databases {*}[glob /mod/etc/*.db]
lappend databases "/mnt/hd2/dms_cds.db"
lappend databases "/mod/tmp/tvdb/series.db"
lappend databases {*}[glob /mod/var/tvdb/*.db]
header

View File

@ -3,7 +3,7 @@
*************/
/* overall */
.tablesorter-webif {
width: 100%;
/*width: 100%;*/
text-align: left;
border-spacing: 0;
border: #cdcdcd 1px solid;

56
webif/html/tvdb/index.jim Executable file
View File

@ -0,0 +1,56 @@
#!/mod/bin/jimsh
source /mod/webif/lib/setup
require ts.class
jqplugin tablesorter2 highlight
jscss script.js
header
puts {
<table>
<thead>
<tr>
<th>Title</th>
<th>Episode</th>
<th>Episode Name</th>
<th>Synopses</th>
</tr>
</thead>
<tbody>
}
ts iterate [lambda {ts} {
set epname [$ts episode_name]
set series [$ts get tvdb_series]
set data [$ts get tvdb_data]
if {[dict exists $data overview]} {
set overview $data(overview)
} else { set overview "" }
puts "
<tr>
<td rowspan=3>[$ts get title]</td>
<td rowspan=3>[$ts epstr]</td>
<td rowspan=3>$epname</td>
<td class=synopsis>[$ts get synopsis]</td>
</tr>
<tr>
<td class=words>
Found by [$ts get tvdb_method]<br>
[$series get _phrases]
</td>
</tr>
<tr>
<td class=synopsis>$overview</td>
</tr>
"
}] 0
puts {
</tbody>
</table>
}
footer

12
webif/html/tvdb/script.js Normal file
View File

@ -0,0 +1,12 @@
$(function() {
$('table')
.tablesorter({
theme: 'webif',
widthFixed: false,
widgets: ['zebra', 'stickyHeaders']
});
});

View File

@ -143,7 +143,7 @@ proc dorecalc {dir} {
if {!$recalc} return
log "Running unwatched recalculation for $dir" 2
ts resetnew $dir
set recalc 0
incr recalc -1
}
proc startop {op file} {

13
webif/lib/bin/epname Executable file
View File

@ -0,0 +1,13 @@
#!/mod/bin/jimsh
source /mod/webif/lib/setup
require system.class ts.class
ts iterate [lambda {ts} {
puts "[$ts get title] - ([$ts series_name])"
puts [$ts get synopsis]
puts [$ts episode_name]
puts [$ts epstr]
puts ""
}] 1

BIN
webif/lib/bin/tvdb Executable file

Binary file not shown.

View File

@ -66,10 +66,14 @@ proc {file tdelete} {target} {
}
}
proc {file read} {target} {
proc {file read} {target {bytes 0}} {
if {[file readable $target]} {
set fd [open $target]
set ret [$fd read]
if {$bytes} {
set ret [$fd read $bytes]
} else {
set ret [$fd read]
}
$fd close
return $ret
}

View File

@ -524,11 +524,12 @@ proc {system usbmounts} {{full 0}} {
proc {system has} {comp} {
switch $comp {
wifi_dongle {
if {[catch {exec /mod/bin/iwgetid}]} {
return 0
}
if {[catch {exec /mod/bin/iwgetid}]} { return 0 }
return 1
}
tvdb {
return [file exists /mod/webif/.tvdb]
}
}
return 0
}

39
webif/lib/test/episode Executable file
View File

@ -0,0 +1,39 @@
#!/mod/bin/jimsh
source /mod/webif/lib/setup
require ts.class
foreach {sample expected} {
{Episode 5} {S0E5/0}
{(8/8)} {S0E8/8}
{(Episode 5/10)} {S0E5/10}
{(Ep5/10)} {S0E5/10}
{(Ep 3 of 3)} {S0E3/3}
{(Ep3)} {S0E3/0}
{(S2 Ep1)} {S2E1/0}
{S.02 Ep.002} {S2E2/0}
{S01 Ep52} {S1E52/0}
{(S4 Ep 7)} {S4E7/0}
{(S1, ep 2)} {S1E2/0}
{(S8, Ep2)} {S8E2/0}
{(S4 Ep22/24)} {S4E22/24}
{23/27.} {S0E23/27}
} {
set ts [ts new "synopsis {$sample}"]
$ts episode_name
set epstr [format "S%dE%d/%d" \
[$ts get seriesnum] \
[$ts get episodenum] \
[$ts get episodetot] \
]
if {$epstr eq $expected} {
set result "OK"
} else {
set result "FAIL"
}
puts [format "%-20s => %-10s - %s" $sample $epstr $result]
}

View File

@ -2,7 +2,7 @@
if {![exists -proc class]} { package require oo }
if {![exists -proc pack]} { package require pack }
source /mod/webif/lib/setup
require system.class
require system.class tvdb.class classdump
set dmsfile /mnt/hd2/dms_cds.db
@ -27,6 +27,15 @@ class ts {
genre 0
resume 0
status ""
series ""
seriesnum 0
episodenum 0
episodetot 0
episodename ""
tvdb_method ""
tvdb_series {}
tvdb_data {}
}
ts method bfile {} {
@ -460,3 +469,159 @@ proc {ts iterate} {callback {verbose 0} {dir ""}} {{rootdev 0}} {
}
}
#
# Attempt to extract the series/episode names using a variety of techniques
#
ts method series_name {} {
# For recorded series, use the folder name
set dir [file dirname $file]
if {[file exists "$dir/.series"]} {
set s [file tail $dir]
} else {
set s $title
}
foreach x {
{^new: *}
} {
regsub -nocase -all -- $x $s "" s
}
return $s
}
set ::ts::episode_prefixes {
{^new series\.* *}
{^cbeebies\.* *}
{^cbbc\.* *}
{^t4: *}
{^brand new series *[-:]* *}
{^\.+}
}
ts method tvdb_resolve {} {
# See if we can find a TVDB series for this recording.
set tvdb_series [set v [tvdb series [$self series_name]]]
if {[$v get seriesid] == 0} { return }
# Got one.
if {$seriesnum && $episodenum} {
# Easiest case - we can explicitly request the episode.
set tvdb_method "series and episode number"
return [$v episodebynum $seriesnum $episodenum]
}
# Now try to find the episode using the current episode name
set k [$v episodebyname $episodename]
if {[llength $k]} {
set tvdb_method "episode name ($episodename)"
return $k
}
if {$episodenum} {
# More problematic but can at least narrow the list of
# candidates.
set tvdb_method "episode number"
return [$v episodebyepnum $episodenum $synopsis]
}
# Most difficult - try and match based on synopsis alone
set tvdb_method "synopsis text"
return [$v episodebysynopsis $synopsis]
}
ts method episode_name {} {
set s $synopsis
######################################################################
# Attempt to determine the episode name from the synopsis
# Strip common prefixes
foreach prefix $::ts::episode_prefixes {
regsub -nocase -all -- $prefix $s "" s
}
# Strip anything following a colon.
regsub -all -- { *[:].*$} $s "" s
# If the resulting string is longer than 40 characters then
# split around . and take the left hand side if appropriate.
if {[string length $s] > 40} {
lassign [split $s "."] v w
set s $v
if {[string length $s] < 6 && [string length $w] < 6} {
append s "_$w"
}
}
# Shorten if too long.
if {[string length $s] > 40} { set s [string range $s 0 39] }
set episodename $s
######################################################################
# Check for embedded Series/Episode number.
# Thank you broadcasters for the variation!
# Least trustworthy first.
# Episode 5
regexp -nocase -- {Episode (\d+)} $synopsis x episodenum
# ^23/27.
regexp -nocase -- {^\s*(\d+)/(\d+)} $synopsis x episodenum episodetot
# (8/8)
regexp -nocase -- {\((\d+)/(\d+)\)} $synopsis x episodenum episodetot
# (Episode 5/10)
# (Ep5/10)
# (Ep 3 of 3)
# (Ep3)
regexp -nocase -- {Epi?s?o?d?e?\s*(\d+)\s*(of|/)?\s*(\d+)?} $synopsis \
x episodenum x episodetot
# (S2 Ep1)
# S.02 Ep.002
# S01 Ep52
# (S4 Ep 7)
# (S1, ep 2)
# (S8, Ep2)
# (S4 Ep22/24)
regexp -nocase -- {S\.*(\d+),?\s*Ep\.?\s*(\d+)(/(\d+))?} $synopsis \
x seriesnum episodenum x episodetot
foreach v {seriesnum episodenum episodetot} {
if {[set $v] eq ""} {
set $v 0
} else {
incr $v 0
}
}
# Now see if TVDB has anything to add
if {[system has tvdb]} {
set tvdb_data [$self tvdb_resolve]
if {[dict exists $tvdb_data name]} {
if {!$seriesnum} { set seriesnum $tvdb_data(series) }
if {!$episodenum} { set episodenum $tvdb_data(episode) }
return $tvdb_data(name)
}
}
return $s
}
ts method epstr {} {
set x s
if {$seriesnum eq 0} { append x "?" } else { append x $seriesnum }
append x e
if {$episodenum eq 0} { append x "??" } else { append x $episodenum }
if {$episodetot ne 0} { append x "/$episodetot" }
return $x
}

252
webif/lib/tvdb.class Normal file
View File

@ -0,0 +1,252 @@
source /mod/webif/lib/setup
package require cgi
if {![exists -proc class]} { package require oo }
if {![exists -proc sqlite3.open]} { package require sqlite3 }
require xml.class
set ::tvdb::apikey 1764335F804A5A91
set ::tvdb::mirror "thetvdb.com"
set ::tvdb::cache "/mod/var/tvdb"
set ::tvdb::cacheage 86400
set ::tvdb::debug 0
if {![file isdirectory $::tvdb::cache]} {
file mkdir $::tvdb::cache
}
set ::tvdb::indexdb [sqlite3.open "$::tvdb::cache/series.db"]
$::tvdb::indexdb query {
create table if not exists nseries(
[name] text primary key
)
}
class tvdb {
seriesid 0
imdb_id ""
name ""
overview ""
dat ""
_matches {}
_smatches {}
_phrases {}
}
proc ::tvdb::dlog {msg} {
if {$::tvdb::debug} {
puts $msg
}
}
tvdb method _fetch {url} {
set f [socket stream $::tvdb::mirror:80]
$f puts -nonewline "GET /api/$url HTTP/1.0\r\n"
$f puts -nonewline "Host: $::tvdb::mirror\r\n"
$f puts -nonewline "Connection: close\r\n"
$f puts -nonewline "\r\n"
set line [string trim [$f gets]]
while {[string length $line]} {
#puts "Web Header: $line"
set line [string trim [$f gets]]
}
# Save the body
set ret [$f read]
$f close
return $ret
}
tvdb method _parse {xml vars {end "XXX"}} {
set x [xml init $xml]
set cattr 0
while {1} {
lassign [$x next] type val attr etype
if {$type == "EOF"} break
switch $etype {
START { set cattr $val }
END {
if {$val == $end} break
set cattr 0
}
default {
if {$type == "TXT"} {
if {$cattr in $vars} {
set $cattr $val
}
::tvdb::dlog "$cattr - $val"
}
}
}
}
}
tvdb method findseries {series} {
set ret ""
catch {
set ret [$::tvdb::indexdb query {
select series_id from series where name = '%s'
} $series]
}
if {[llength $ret]} {
lassign [lindex $ret 0] x seriesid
::tvdb::dlog "Cached ID for '$series' = $seriesid"
}
if {!$seriesid} {
# Attempt to determine series ID
::tvdb::dlog "Trying to find series in TVDB"
set seriesxml [$self _fetch \
"/GetSeries.php?seriesname=[cgi_quote_url $series]"]
$self _parse $seriesxml {seriesid} Series
}
#puts "SERIESID: ($seriesid)"
if {!$seriesid} {
# Negative caching
$::tvdb::indexdb query {
replace into nseries values('%s')
} $series
return;
}
# Found a series
set base "$::tvdb::cache/$seriesid"
# Expire old files
if {[file exists "$base.xml"]} {
set age $([clock seconds] - [file mtime "$base.xml"])
::tvdb::dlog "$base.xml age $age"
if {$age > $::tvdb::cacheage} {
file delete "$base.xml"
::tvdb::dlog "Expiring $base.xml"
}
}
if {![file exists "$base.xml"]} {
::tvdb::dlog "Downloading"
# Download the series info
set f [open "$base.zip" w]
puts $f [$self _fetch \
"$::tvdb::apikey/series/$seriesid/all/en.zip"]
$f close
exec unzip -o -q "$base.zip" en.xml -d $::tvdb::cache
if {[file exists "$::tvdb::cache/en.xml"]} {
file rename "$::tvdb::cache/en.xml" $base.xml
# Extract episode info
puts [exec /mod/webif/lib/bin/tvdb "$base.xml"]
}
}
set ret [$::tvdb::indexdb query {
select * from series where series_id = %s} $seriesid]
if {[llength $ret] == 1} {
foreach {k v} [lindex $ret 0] {
set $k $v
}
}
}
tvdb method dbhandle {} {
if {![file exists "$::tvdb::cache/$seriesid.db"]} { return {} }
set db [sqlite3.open "$::tvdb::cache/$seriesid.db"]
return $db
}
proc {tvdb tolike} {str} {
return "%[string map {
" and " "%"
" the " "%"
} $str]%"
}
tvdb method episodebynum {series episode} {
if {![llength [set db [$self dbhandle]]]} return {}
set ret [$db query {
select * from episode where series = %s and episode = %s
} $series $episode]
if {[llength $ret]} { return [lindex $ret 0] }
return {}
}
tvdb method episodebyname {str} {
if {![llength [set db [$self dbhandle]]]} return {}
set ret [$db query {
select * from episode where name like '%s'
order by length(name)
} [tvdb tolike $str]]
if {[llength $ret]} { return [lindex $ret 0] }
return {}
}
tvdb method wordcount {db words extra} {
set _matches {}
loop i 5 1 -1 {
#puts "Trying phrases of length $i<br>"
set wlen $([llength $words] - $i - 1)
for {set a 0} {$a < $wlen} {incr a} {
set w [lrange $words $a $($a + $i - 1)]
#puts "Phrase: $w<br>"
set ret [$db query {
select episode_id from episode
where overview like '%%%s%%' %s
} $w $extra]
foreach row $ret {
#puts "&nbsp;&nbsp;> FOUND: ($w)<br>"
ladd _phrases $w
lassign $row x id
# Score is twice phrase length
incr _matches($id) $($i * 2)
}
}
}
}
tvdb method seek {synopsis {extra ""}} {
if {![llength [set db [$self dbhandle]]]} return {}
set words [split $synopsis " "]
# Count phrase occurence
$self wordcount $db $words $extra
if {![llength $_matches]} {
return {}
}
# Find best match
set _smatches [lsort -integer -decreasing -index end [\
lmap {k v} $_matches {
list $k $v
}]]
set eid [lindex [lindex $_smatches 0] 0]
set ret [$db query {
select * from episode where episode_id = %s
} $eid]
if {[llength $ret]} { return [lindex $ret 0] }
return {}
}
tvdb method episodebyepnum {episode synopsis} {
return [$self seek $synopsis "and episode = $episode"]
}
tvdb method episodebysynopsis {synopsis} {
return [$self seek $synopsis]
}
proc {tvdb series} {series} {
set t [tvdb]
$t findseries $series
return $t
}

63
webif/lib/xml.class Normal file
View File

@ -0,0 +1,63 @@
# Simple XML parser
if {![exists -proc class]} { package require oo }
class xml {
xml ""
loc 0
}
proc {xml init} {xml} {
# Remove all XML comments
regsub -all {<!--.*?-->} $xml {} xml
return [xml new [list xml [string trim $xml] loc 0]]
}
xml method next {{peek 0}} {
set n [regexp -start $loc -indices {(.*?)\s*?<(/?)(.*?)(/?)>} \
$xml all txt stok tok etok]
if {!$n} {
return "EOF"
}
lassign $all all0 all1
lassign $txt txt0 txt1
lassign $stok stok0 stok1
lassign $tok tok0 tok1
lassign $etok etok0 etok1
if {$txt1 >= $txt0} {
set txt [string range $xml $txt0 $txt1]
if {!$peek} {
set loc [expr {$txt1 + 1}]
}
return [list TXT $txt]
}
set token [string range $xml $tok0 $tok1]
if {!$peek} {
set loc [expr {$all1 + 1}]
}
if {[regexp {^!\[CDATA\[(.*)\]\]} $token => txt]} {
return [list TXT $txt]
}
# Check for Processing Instruction <?...?>
set type XML
if {[regexp {^\?(.*)\?$} $token => token]} {
set type PI
}
set attr ""
regexp {^(.*?)\s+(.*?)$} $token => token attr
set etype START
if {$etok0 <= $etok1} {
if {$stok0 <= $stok1} {
# Bad XML
set token "/$token"
}
set etype EMPTY
} elseif {$stok0 <= $stok1} {
set etype END
}
return [list $type $token $attr $etype]
}