with some 11 exceptions my routine could glue all the threads together again.
this is still far from being elegant, mind you....

this code handles all the exporting from rslight to vichan.
-it checks the articles in the spool,
-builds an index file and a watermark file (if necessary),
-then resolves all the broken threads (fixed 10 iterations, have to make that dynamic later),
-then exports everything to vichan.

on the vichan side there is a modified post.php to get the messages.

so far the three parts (rslight, vichan and the script) are fully decoupled and individually called from cron.

in the long run code could be merged of course.

import script:

<?php

include "config.inc.php";
include "$file_newsportal";
include 'message.inc.php';

$groups = array(
'rocksolid.nodes',
'rocksolid.nodes.help',
'rocksolid.nodes.announce',
'rocksolid.shared.encryption',
'rocksolid.shared.entertainment',
'rocksolid.shared.freenet',
'rocksolid.shared.general',
'rocksolid.shared.hacking',
'rocksolid.shared.helpdesk',
'rocksolid.shared.i2p',
'rocksolid.shared.linux',
'rocksolid.shared.news',
'rocksolid.shared.offtopic',
'rocksolid.programming',
'rocksolid.shared.rocksolid',
'rocksolid.shared.security',
'rocksolid.social',
'rocksolid.shared.test',
'rocksolid.shared.tor',
);

if(!file_exists('xt.json')){
echo("no index found, building it \n\r");
$threads = build_threads($groups);
} else {
$threads = json_decode(file_get_contents('xt.json'), true);
$watermarks = json_decode(file_get_contents('xw.json'), true);
$threads = update_threads($groups, $threads, $watermarks);
} $thread_keys = array_keys($threads);
for ($i = 0; $i < 10; $i++){
$threads = stage($thread_keys, $threads);
}

$jsonFile = fopen('xt.json', 'w');
fwrite($jsonFile, json_encode($threads, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
fclose($jsonFile);

update_sync($threads);

exit;

function build_threads($groups){
$threads = array();
$watermarks = array();
foreach($groups as $group){
$watermarks[$group] = array();
echo("building index for group " . $group . "\n\r");
for ($i = 0; $i < 1000; $i++){
$message = message_read($i, 0, $group);
if($message){
echo("processing message # " . $i . " from group " . $group . "\r");
$rawData = json_decode(json_encode((array) $message, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), true);
$ID = $rawData['header']['id'];
$Ref = $rawData['header']['references']['0'];
$firstRef = explode("<", $Ref);
$Ref = ("<" . $firstRef[1]);
$threads[$ID] = array();
$threads[$ID]['parents'] = array();
$threads[$ID]['synced'] = false;
$threads[$ID]['number'] = $i;
$threads[$ID]['group'] = $group;
$threads[$ID]['message'] = array();
$threads[$ID]['message'] = $rawData;
if($Ref == "<"){
$threads[$ID]['op'] = true;
$threads[$ID]['solved'] = true;
} else {
$threads[$ID]['op'] = false;
$threads[$ID]['ref'] = $Ref;
$threads[$ID]['ref_org'] = $rawData['header']['references']['0'];
$threads[$ID]['solved'] = false;
}
array_push($watermarks[$group],$i);
}
}
}
$jsonFile = fopen('xt.json', 'w');
fwrite($jsonFile, json_encode($threads, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
fclose($jsonFile);
$jsonWatermarks = fopen('xw.json', 'w');
fwrite($jsonWatermarks, json_encode((array) $watermarks, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
fclose($jsonWatermarks);
return $threads;
} function update_threads($groups, $threads, $watermarks){
$update = false;
foreach($groups as $group){
echo("updating index for group " . $group . "\n\r");
for ($i = 0; $i < 1000; $i++){
if(!in_array($i,$watermarks[$group])){
$message = message_read($i, 0, $group);
if($message){
echo("processing message # " . $i . " from group " . $group . "\r");
$update = true;
$rawData = json_decode(json_encode((array) $message, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), true);
$ID = $rawData['header']['id'];
$Ref = $rawData['header']['references']['0'];
$firstRef = explode("<", $Ref);
$Ref = ("<" . $firstRef[1]);
$threads[$ID] = array();
$threads[$ID]['parents'] = array();
$threads[$ID]['synced'] = false;
$threads[$ID]['number'] = $i;
$threads[$ID]['group'] = $group;
$threads[$ID]['message'] = array();
$threads[$ID]['message'] = $rawData;
if($Ref == "<"){
$threads[$ID]['op'] = true;
$threads[$ID]['solved'] = true;
} else {
$threads[$ID]['op'] = false;
$threads[$ID]['ref'] = $Ref;
$threads[$ID]['ref_org'] = $rawData['header']['references']['0'];
$threads[$ID]['solved'] = false;
}
array_push($watermarks[$group],$i);
}
}
}
}
if($update){
echo("saving updated threading\n\r");
$jsonFile = fopen('xt.json', 'w');
fwrite($jsonFile, json_encode($threads, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
fclose($jsonFile);
echo("saving updated watermarks\n\r");
$jsonWatermarks = fopen('xw.json', 'w');
fwrite($jsonWatermarks, json_encode((array) $watermarks, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
fclose($jsonWatermarks);
} else {
echo("no new files found\n\r");
}
return $threads;
}

function stage($thread_keys, $threads){
foreach($thread_keys as $thread_key){
if( ($threads[$thread_key]['solved'] !== true) && (threads[$thread_key]['op'] !== true) ){
echo("checking message" . $thread_key . "\r");
if ($threads[$thread_key]['parents']){
$to_check = end($threads[$thread_key]['parents']);
if(!array_key_exists($to_check, $threads)){
$threads[$thread_key]['op'] = true;
$threads[$thread_key]['ref'] = array();
$threads[$thread_key]['missing_ref'] = $to_check;
$threads[$thread_key]['solved'] = true;
} else {
if($threads[$to_check]['op'] == true){
$threads[$thread_key]['ref'] = $to_check;
$threads[$thread_key]['solved'] = true;
} else {
array_push($threads[$thread_key]['parents'], $threads[$to_check]['ref']);
array_filter($threads[$thread_key]['parents'], static function($var){return $var !== null;} );
}
}
} elseif ($threads[$thread_key]['ref']){
array_push($threads[$thread_key]['parents'], $threads[$thread_key]['ref']);
}
}
}
return $threads;
}

function update_sync($threads){
$update = false;
$thread_keys = array_keys($threads);
foreach($thread_keys as $thread_key){
if( ($threads[$thread_key]['synced'] == false) && ($threads[$thread_key]['solved'] == true) ){
echo("processing message # " . $threads[$thread_key]['number'] . " from group " . $threads[$thread_key]['group'] . "\n\r");
$rawData = $threads[$thread_key]['message'];
if($threads[$thread_key]['ref']){
$rawData['header']['references']['0'] = $threads[$thread_key]['ref'];
} else {
unset($rawData['header']['references']);
}
$paddedI = str_pad($threads[$thread_key]['number'], 5, '0', STR_PAD_LEFT);
$jsonFile = fopen('/var/www/html/vi/vichan-master/debug/' . $paddedI . '_' . $threads[$thread_key]['group'] . '.json', 'w');
fwrite($jsonFile, json_encode((array) $rawData, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
fclose($jsonFile);
chown($jsonFile, 'www-data');
chgrp($jsonFile, 'www-data');
$update = true;
$threads[$thread_key]['synced'] = true;
echo($thread['synced'] . "\n\r");
}
}
if($update){
echo("saving updated syncing\n\r");
$jsonFile = fopen('xt.json', 'w');
fwrite($jsonFile, json_encode($threads, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
fclose($jsonFile);
} else {
echo("no new files found for syncing\n\r");
}
} ?>

cheers

trw