davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Add a timeout of 2 seconds for each online check to make sure the online check doesn't get blocked somehow

This commit is contained in:
0xb00bface 2022-04-10 17:47:40 +02:00
parent a008fff084
commit 04eb5a7ad1
3 changed files with 191 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/src/main/resources/logback.xml
View File

@ -45,7 +45,7 @@
<logger name="ctbrec.LoggingInterceptor" level="INFO"/> <logger name="ctbrec.LoggingInterceptor" level="INFO"/>
<logger name="ctbrec.io.CookieJarImpl" level="INFO"/> <logger name="ctbrec.io.CookieJarImpl" level="INFO"/>
<logger name="ctbrec.recorder.FFmpeg" level="DEBUG"/> <logger name="ctbrec.recorder.FFmpeg" level="DEBUG"/>
<logger name="ctbrec.recorder.OnlineMonitor" level="INFO"/> <logger name="ctbrec.recorder.OnlineMonitor" level="DEBUG"/>
<logger name="ctbrec.recorder.RecordingFileMonitor" level="TRACE"/> <logger name="ctbrec.recorder.RecordingFileMonitor" level="TRACE"/>
<logger name="ctbrec.recorder.download.dash.DashDownload" level="DEBUG"/> <logger name="ctbrec.recorder.download.dash.DashDownload" level="DEBUG"/>
<logger name="ctbrec.recorder.server.HlsServlet" level="INFO"/> <logger name="ctbrec.recorder.server.HlsServlet" level="INFO"/>

108
common/src/main/java/ctbrec/recorder/OnlineMonitor.java
View File

@ -1,44 +1,38 @@
package ctbrec.recorder; package ctbrec.recorder;
import static ctbrec.Model.State.*;
import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.Config; import ctbrec.Config;
import ctbrec.Model; import ctbrec.Model;
import ctbrec.event.EventBusHolder; import ctbrec.event.EventBusHolder;
import ctbrec.event.ModelIsOnlineEvent; import ctbrec.event.ModelIsOnlineEvent;
import ctbrec.event.ModelStateChangedEvent; import ctbrec.event.ModelStateChangedEvent;
import ctbrec.io.HttpException; import ctbrec.io.HttpException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
import java.time.Duration;
import java.time.Instant;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import static ctbrec.Model.State.UNKNOWN;
public class OnlineMonitor extends Thread { public class OnlineMonitor extends Thread {
private static final Logger LOG = LoggerFactory.getLogger(OnlineMonitor.class); private static final Logger LOG = LoggerFactory.getLogger(OnlineMonitor.class);
private static final boolean IGNORE_CACHE = true; private static final boolean IGNORE_CACHE = true;
private static final long TIMEOUT_IN_MILLIS = 2000;
private volatile boolean running = false; private volatile boolean running = false;
private Recorder recorder; private final Recorder recorder;
private Map<Model, Model.State> states = new HashMap<>(); private final Map<Model, Model.State> states = new HashMap<>();
private Map<String, ExecutorService> executors = new HashMap<>(); private final Map<String, ExecutorService> executors = new HashMap<>();
private Config config; private final Config config;
public OnlineMonitor(Recorder recorder, Config config) { public OnlineMonitor(Recorder recorder, Config config) {
this.recorder = recorder; this.recorder = recorder;
@ -68,19 +62,14 @@ public class OnlineMonitor extends Thread {
} }
private void removeDeletedModels(List<Model> models) { private void removeDeletedModels(List<Model> models) {
for (Iterator<Model> iterator = states.keySet().iterator(); iterator.hasNext();) { states.keySet().removeIf(model -> !models.contains(model));
Model model = iterator.next();
if(!models.contains(model)) {
iterator.remove();
}
}
} }
private void updateModels(List<Model> models) { private void updateModels(List<Model> models) {
// sort models by priority // sort models by priority
Collections.sort(models, (a, b) -> b.getPriority() - a.getPriority()); models.sort((a, b) -> b.getPriority() - a.getPriority());
// submit online check jobs to the executor for the model's site // submit online check jobs to the executor for the model's site
List<Future<?>> futures = new LinkedList<>(); List<ModelAwareFuture> futures = new LinkedList<>();
for (Model model : models) { for (Model model : models) {
boolean skipCheckForSuspended = config.getSettings().onlineCheckSkipsPausedModels && model.isSuspended(); boolean skipCheckForSuspended = config.getSettings().onlineCheckSkipsPausedModels && model.isSuspended();
boolean skipCheckForMarkedAsLater = model.isMarkedForLaterRecording(); boolean skipCheckForMarkedAsLater = model.isMarkedForLaterRecording();
@ -91,18 +80,21 @@ public class OnlineMonitor extends Thread {
} }
} }
// wait for all jobs to finish // wait for all jobs to finish
for (Future<?> future : futures) { for (ModelAwareFuture future : futures) {
try { try {
future.get(); future.get(TIMEOUT_IN_MILLIS, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) { } catch (InterruptedException e) {
LOG.debug("Online check interrupted for model {}", future.getModel(), e);
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} catch (ExecutionException e) { } catch (ExecutionException e) {
LOG.info("Error while checking online state", e); LOG.info("Error while checking online state for model {}", future.getModel(), e);
} catch (TimeoutException e) {
LOG.debug("Online check didn't finish after {}ms for model {}", TIMEOUT_IN_MILLIS, future.getModel());
} }
} }
} }
private Future<?> updateModel(Model model) { private ModelAwareFuture updateModel(Model model) {
final String siteName = model.getSite().getName(); final String siteName = model.getSite().getName();
ExecutorService executor = executors.computeIfAbsent(siteName, name -> Executors.newSingleThreadExecutor(r -> { ExecutorService executor = executors.computeIfAbsent(siteName, name -> Executors.newSingleThreadExecutor(r -> {
Thread t = new Thread(r); Thread t = new Thread(r);
@ -112,14 +104,14 @@ public class OnlineMonitor extends Thread {
return t; return t;
})); }));
return executor.submit(() -> { var future = executor.submit(() -> {
try { try {
if (model.isOnline(IGNORE_CACHE)) { if (model.isOnline(IGNORE_CACHE)) {
EventBusHolder.BUS.post(new ModelIsOnlineEvent(model)); EventBusHolder.BUS.post(new ModelIsOnlineEvent(model));
model.setLastSeen(Instant.now()); model.setLastSeen(Instant.now());
} }
Model.State state = model.getOnlineState(false); Model.State state = model.getOnlineState(false);
LOG.trace("Model online state: {} {}", model.getName(), state); LOG.debug("Model online state: {} {}", model.getName(), state);
Model.State oldState = states.getOrDefault(model, UNKNOWN); Model.State oldState = states.getOrDefault(model, UNKNOWN);
states.put(model, state); states.put(model, state);
if (state != oldState) { if (state != oldState) {
@ -137,13 +129,15 @@ public class OnlineMonitor extends Thread {
} catch (Exception e) { } catch (Exception e) {
LOG.error("Couldn't check if model {} is online", model.getName(), e); LOG.error("Couldn't check if model {} is online", model.getName(), e);
} }
return model;
}); });
return new ModelAwareFuture(model, future);
} }
private void suspendUntilNextIteration(List<Model> models, Duration timeCheckTook) { private void suspendUntilNextIteration(List<Model> models, Duration timeCheckTook) {
LOG.debug("Online check for {} models took {} seconds", models.size(), timeCheckTook.getSeconds()); LOG.debug("Online check for {} models took {} seconds", models.size(), timeCheckTook.getSeconds());
long sleepTime = config.getSettings().onlineCheckIntervalInSecs; long sleepTime = config.getSettings().onlineCheckIntervalInSecs;
if(timeCheckTook.getSeconds() < sleepTime) { if (timeCheckTook.getSeconds() < sleepTime) {
try { try {
if (running) { if (running) {
long millis = TimeUnit.SECONDS.toMillis(sleepTime - timeCheckTook.getSeconds()); long millis = TimeUnit.SECONDS.toMillis(sleepTime - timeCheckTook.getSeconds());
@ -164,4 +158,36 @@ public class OnlineMonitor extends Thread {
} }
interrupt(); interrupt();
} }
}
private record ModelAwareFuture(Model model, Future<Model> future) implements Future<Model> {
public Model getModel() {
return model;
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
return future.cancel(mayInterruptIfRunning);
}
@Override
public boolean isCancelled() {
return future.isCancelled();
}
@Override
public boolean isDone() {
return future.isDone();
}
@Override
public Model get() throws InterruptedException, ExecutionException {
return future.get();
}
@Override
public Model get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
return future.get(timeout, unit);
}
}
}

235
common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java
View File

@ -20,6 +20,7 @@ import java.net.URLDecoder;
import java.util.*; import java.util.*;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer; import java.util.function.Consumer;
@ -125,8 +126,8 @@ public class MyFreeCamsClient {
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE) .header(CONNECTION, KEEP_ALIVE)
.build(); .build();
try(Response resp = mfc.getHttpClient().execute(req)) { try (Response resp = mfc.getHttpClient().execute(req)) {
if(!resp.isSuccessful()) { if (!resp.isSuccessful()) {
throw new HttpException(resp.code(), resp.message()); throw new HttpException(resp.code(), resp.message());
} }
} }
@ -208,93 +209,93 @@ public class MyFreeCamsClient {
try { try {
while ((message = parseMessage(msgBuffer)) != null) { while ((message = parseMessage(msgBuffer)) != null) {
switch (message.getType()) { switch (message.getType()) {
case NULL: case NULL:
LOG.trace("NULL websocket still alive"); LOG.trace("NULL websocket still alive");
break; break;
case LOGIN: case LOGIN:
LOG.debug("LOGIN: {}", message); LOG.debug("LOGIN: {}", message);
sessionId = message.getReceiver(); sessionId = message.getReceiver();
LOG.debug("Session ID {}", sessionId); LOG.debug("Session ID {}", sessionId);
break; break;
case DETAILS: case DETAILS:
case ROOMHELPER: case ROOMHELPER:
case ADDFRIEND: case ADDFRIEND:
case ADDIGNORE: case ADDIGNORE:
case CMESG: case CMESG:
case PMESG: case PMESG:
case TXPROFILE: case TXPROFILE:
case MYCAMSTATE: case MYCAMSTATE:
case MYWEBCAM: case MYWEBCAM:
case JOINCHAN: case JOINCHAN:
case SESSIONSTATE: case SESSIONSTATE:
if (!message.getMessage().isEmpty()) { if (!message.getMessage().isEmpty()) {
//LOG.debug("SessionState: {}", message.getMessage()); //LOG.debug("SessionState: {}", message.getMessage());
JsonAdapter<SessionState> adapter = moshi.adapter(SessionState.class); JsonAdapter<SessionState> adapter = moshi.adapter(SessionState.class);
try { try {
SessionState sessionState = adapter.fromJson(message.getMessage()); SessionState sessionState = adapter.fromJson(message.getMessage());
updateSessionState(sessionState); updateSessionState(sessionState);
} catch (IOException e) { } catch (IOException e) {
LOG.error("Couldn't parse session state message {}", message, e); LOG.error("Couldn't parse session state message {}", message, e);
}
} }
} break;
break; case USERNAMELOOKUP:
case USERNAMELOOKUP: // LOG.debug("{}", message.getType());
// LOG.debug("{}", message.getType()); // LOG.debug("{}", message.getSender());
// LOG.debug("{}", message.getSender()); // LOG.debug("{}", message.getReceiver());
// LOG.debug("{}", message.getReceiver()); // LOG.debug("{}", message.getArg1());
// LOG.debug("{}", message.getArg1()); // LOG.debug("{}", message.getArg2());
// LOG.debug("{}", message.getArg2()); // LOG.debug("{}", message.getMessage());
// LOG.debug("{}", message.getMessage()); Consumer<Message> responseHandler = responseHandlers.remove(message.getArg1());
Consumer<Message> responseHandler = responseHandlers.remove(message.getArg1()); if (responseHandler != null) {
if (responseHandler != null) { responseHandler.accept(message);
responseHandler.accept(message);
}
break;
case TAGS:
JSONObject json = new JSONObject(message.getMessage());
String[] names = JSONObject.getNames(json);
Integer uid = Integer.parseInt(names[0]);
SessionState sessionState = sessionStates.getIfPresent(uid);
if (sessionState != null) {
JSONArray tags = json.getJSONArray(names[0]);
for (Object obj : tags) {
sessionState.getM().getTags().add((String) obj);
} }
} break;
break; case TAGS:
case EXTDATA: JSONObject json = new JSONObject(message.getMessage());
if (message.getArg1() == MessageTypes.LOGIN) { String[] names = JSONObject.getNames(json);
// noop Integer uid = Integer.parseInt(names[0]);
} else if (message.getArg1() == MessageTypes.MANAGELIST) { SessionState sessionState = sessionStates.getIfPresent(uid);
requestExtData(message.getMessage()); if (sessionState != null) {
} else { JSONArray tags = json.getJSONArray(names[0]);
LOG.debug("EXTDATA: {}", message); for (Object obj : tags) {
} sessionState.getM().getTags().add((String) obj);
break; }
case ROOMDATA: }
LOG.debug("ROOMDATA: {}", message); break;
break; case EXTDATA:
case UEOPT: if (message.getArg1() == MessageTypes.LOGIN) {
LOG.trace("UEOPT: {}", message); // noop
break; } else if (message.getArg1() == MessageTypes.MANAGELIST) {
case SLAVEVSHARE: requestExtData(message.getMessage());
// LOG.debug("SLAVEVSHARE {}", message); } else {
// LOG.debug("SLAVEVSHARE MSG [{}]", message.getMessage()); LOG.debug("EXTDATA: {}", message);
break; }
case TKX: break;
json = new JSONObject(message.getMessage()); case ROOMDATA:
tkx = json.getString("tkx"); LOG.debug("ROOMDATA: {}", message);
cxid = json.getInt("cxid"); break;
ctxenc = URLDecoder.decode(json.getString("ctxenc"), UTF_8); case UEOPT:
JSONArray ctxArray = json.getJSONArray("ctx"); LOG.trace("UEOPT: {}", message);
ctx = new int[ctxArray.length()]; break;
for (int i = 0; i < ctxArray.length(); i++) { case SLAVEVSHARE:
ctx[i] = ctxArray.getInt(i); // LOG.debug("SLAVEVSHARE {}", message);
} // LOG.debug("SLAVEVSHARE MSG [{}]", message.getMessage());
break; break;
default: case TKX:
LOG.trace("Unknown message {}", message); json = new JSONObject(message.getMessage());
break; tkx = json.getString("tkx");
cxid = json.getInt("cxid");
ctxenc = URLDecoder.decode(json.getString("ctxenc"), UTF_8);
JSONArray ctxArray = json.getJSONArray("ctx");
ctx = new int[ctxArray.length()];
for (int i = 0; i < ctxArray.length(); i++) {
ctx[i] = ctxArray.getInt(i);
}
break;
default:
LOG.trace("Unknown message {}", message);
break;
} }
} }
} catch (Exception e) { } catch (Exception e) {
@ -598,7 +599,7 @@ public class MyFreeCamsClient {
for (SessionState state : sessionStates.asMap().values()) { for (SessionState state : sessionStates.asMap().values()) {
Optional<String> nm = Optional.ofNullable(state.getNm()); Optional<String> nm = Optional.ofNullable(state.getNm());
Optional<String> name = Optional.ofNullable(model.getName()); Optional<String> name = Optional.ofNullable(model.getName());
if(nm.isEmpty() || name.isEmpty()) { if (nm.isEmpty() || name.isEmpty()) {
continue; continue;
} }
@ -672,36 +673,46 @@ public class MyFreeCamsClient {
LOG.trace("Sending USERNAMELOOKUP for {}", q); LOG.trace("Sending USERNAMELOOKUP for {}", q);
Object monitor = new Object(); Object monitor = new Object();
int msgId = messageId++; int msgId = messageId++;
AtomicBoolean searchDone = new AtomicBoolean(false);
responseHandlers.put(msgId, msg -> { responseHandlers.put(msgId, msg -> {
LOG.trace("Search result: {}", msg); try {
if (StringUtil.isNotBlank(msg.getMessage()) && !Objects.equals(msg.getMessage(), q)) { LOG.trace("Search result: {}", msg);
JSONObject json = new JSONObject(msg.getMessage()); if (StringUtil.isNotBlank(msg.getMessage()) && !Objects.equals(msg.getMessage(), q)) {
JSONObject json = new JSONObject(msg.getMessage());
JsonAdapter<SessionState> adapter = moshi.adapter(SessionState.class); JsonAdapter<SessionState> adapter = moshi.adapter(SessionState.class);
try { try {
SessionState sessionState = Objects.requireNonNull(adapter.fromJson(msg.getMessage())); SessionState sessionState = Objects.requireNonNull(adapter.fromJson(msg.getMessage()));
updateSessionState(sessionState); updateSessionState(sessionState);
} catch (Exception e) { } catch (Exception e) {
LOG.error("Couldn't parse session state message {}", msg, e); LOG.error("Couldn't parse session state message {}", msg, e);
}
String name = json.getString("nm");
MyFreeCamsModel model = mfc.createModel(name);
model.setUid(json.getInt("uid"));
model.setMfcState(State.of(json.getInt("vs")));
String uid = Integer.toString(model.getUid());
String uidStart = uid.substring(0, 3);
String previewUrl = "https://img.mfcimg.com/photos2/" + uidStart + '/' + uid + "/avatar.90x90.jpg";
model.setPreview(previewUrl);
result.add(model);
}
} finally {
searchDone.setPlain(true);
synchronized (monitor) {
monitor.notifyAll();
} }
String name = json.getString("nm");
MyFreeCamsModel model = mfc.createModel(name);
model.setUid(json.getInt("uid"));
model.setMfcState(State.of(json.getInt("vs")));
String uid = Integer.toString(model.getUid());
String uidStart = uid.substring(0, 3);
String previewUrl = "https://img.mfcimg.com/photos2/" + uidStart + '/' + uid + "/avatar.90x90.jpg";
model.setPreview(previewUrl);
result.add(model);
}
synchronized (monitor) {
monitor.notifyAll();
} }
}); });
ws.send("10 " + sessionId + " 0 " + msgId + " 0 " + q + "\n"); ws.send("10 " + sessionId + " 0 " + msgId + " 0 " + q + "\n");
synchronized (monitor) { int waitInMillis = 1000;
monitor.wait(); int iterations = 0;
while (iterations < 5 && !searchDone.get()) {
synchronized (monitor) {
monitor.wait(waitInMillis);
iterations++;
}
} }
for (MyFreeCamsModel model : models.asMap().values()) { for (MyFreeCamsModel model : models.asMap().values()) {