From f11fcf7ca1acd4b06672dcd3ce0f9c310d0c67ef Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sat, 1 Jun 2019 12:12:46 +0200 Subject: [PATCH] Rewrite recording code for remote recording --- .../main/java/ctbrec/ui/JavaFxRecording.java | 4 + .../main/java/ctbrec/ui/RecordingsTab.java | 4 +- common/src/main/java/ctbrec/Recording.java | 56 +-- .../ctbrec/recorder/NextGenLocalRecorder.java | 4 +- .../java/ctbrec/recorder/OnlineMonitor.java | 2 + .../ctbrec/recorder/RecordingManager.java | 3 + .../java/ctbrec/recorder/RemoteRecorder.java | 37 +- .../ctbrec/recorder/download/Download.java | 2 + .../ctbrec/recorder/download/HlsDownload.java | 90 +++-- .../recorder/download/MergedHlsDownload.java | 11 + .../jasmin/LiveJasminChunkedHttpDownload.java | 304 --------------- .../jasmin/LiveJasminWebSocketDownload.java | 368 ------------------ .../ctbrec/recorder/server/HttpServer.java | 4 +- .../recorder/server/RecorderServlet.java | 9 +- 14 files changed, 117 insertions(+), 781 deletions(-) delete mode 100644 common/src/main/java/ctbrec/sites/jasmin/LiveJasminChunkedHttpDownload.java delete mode 100644 common/src/main/java/ctbrec/sites/jasmin/LiveJasminWebSocketDownload.java diff --git a/client/src/main/java/ctbrec/ui/JavaFxRecording.java b/client/src/main/java/ctbrec/ui/JavaFxRecording.java index b9265034..7062bf0b 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxRecording.java +++ b/client/src/main/java/ctbrec/ui/JavaFxRecording.java @@ -26,6 +26,10 @@ public class JavaFxRecording extends Recording { setProgress(recording.getProgress()); } + public Recording getDelegate() { + return delegate; + } + @Override public Model getModel() { return delegate.getModel(); diff --git a/client/src/main/java/ctbrec/ui/RecordingsTab.java b/client/src/main/java/ctbrec/ui/RecordingsTab.java index 68d3b54a..208a5909 100644 --- a/client/src/main/java/ctbrec/ui/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/RecordingsTab.java @@ -525,7 +525,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener { }.start(); } else { String hlsBase = "http://" + config.getSettings().httpServer + ":" + config.getSettings().httpPort + "/hls"; - url = hlsBase + "/" + recording.getPath() + "/playlist.m3u8"; + url = hlsBase + recording.getPath() + "/playlist.m3u8"; new Thread() { @Override public void run() { @@ -567,7 +567,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener { continue; } try { - recorder.delete(r); + recorder.delete(r.getDelegate()); deleted.add(r); } catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) { LOG.error("Error while deleting recording", e1); diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index 83525d04..b61b3e47 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -1,8 +1,6 @@ package ctbrec; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.time.Duration; import java.time.Instant; @@ -10,15 +8,8 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import com.iheartradio.m3u8.Encoding; -import com.iheartradio.m3u8.Format; import com.iheartradio.m3u8.ParseException; -import com.iheartradio.m3u8.ParsingMode; import com.iheartradio.m3u8.PlaylistException; -import com.iheartradio.m3u8.PlaylistParser; -import com.iheartradio.m3u8.data.MediaPlaylist; -import com.iheartradio.m3u8.data.Playlist; -import com.iheartradio.m3u8.data.TrackData; import ctbrec.event.EventBusHolder; import ctbrec.event.RecordingStateChangedEvent; @@ -61,15 +52,6 @@ public class Recording { public Recording() {} - // public Recording(String path) throws ParseException { - // this.path = path; - // this.modelName = path.substring(0, path.indexOf("/")); - // SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm"); - // Date date = sdf.parse(path.substring(path.indexOf('/')+1)); - // startDate = Instant.ofEpochMilli(date.getTime()); - // } - - public Instant getStartDate() { return startDate; } @@ -158,42 +140,10 @@ public class Recording { } public Duration getLength() throws IOException, ParseException, PlaylistException { - // check, if the recording exists - File rec = new File(Config.getInstance().getSettings().recordingsDir, getPath()); - if (!rec.exists()) { - return Duration.ofSeconds(0); - } - - // check, if the recording has data at all - long size = getSizeInByte(); - if (size == 0) { - return Duration.ofSeconds(0); - } - - // determine the length - if (getPath().endsWith(".ts")) { - return Duration.ofSeconds((long) MpegUtil.getFileDuration(rec)); - } else if (rec.isDirectory()) { - File playlist = new File(rec, "playlist.m3u8"); - if (playlist.exists()) { - return Duration.ofSeconds((long) getPlaylistLength(playlist)); - } - } - return Duration.ofSeconds(0); - } - - private double getPlaylistLength(File playlist) throws IOException, ParseException, PlaylistException { - if (playlist.exists()) { - PlaylistParser playlistParser = new PlaylistParser(new FileInputStream(playlist), Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); - Playlist m3u = playlistParser.parse(); - MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist(); - double length = 0; - for (TrackData trackData : mediaPlaylist.getTracks()) { - length += trackData.getTrackInfo().duration; - } - return length; + if (getDownload() != null) { + return getDownload().getLength(); } else { - throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist"); + return Duration.ofSeconds(0); } } diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index a91ea2e2..22c55a75 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -368,7 +368,9 @@ public class NextGenLocalRecorder implements Recorder { recording = false; LOG.debug("Stopping all recording processes"); - for (Recording rec : recordingProcesses.values()) { + // make a copy to avoid ConcurrentModificationException + List toStop = new ArrayList<>(recordingProcesses.values()); + for (Recording rec : toStop) { Optional.ofNullable(rec.getDownload()).ifPresent(Download::stop); } diff --git a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java index ab4088e1..796cd6d9 100644 --- a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java +++ b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java @@ -31,6 +31,8 @@ public class OnlineMonitor extends Thread { private Map states = new HashMap<>(); + // TODO divide models into buckets by their site in each iteration a model of each bucket can be testes in parallel + // this will speed up the testing, but not hammer the sites public OnlineMonitor(Recorder recorder) { this.recorder = recorder; setName("OnlineMonitor"); diff --git a/common/src/main/java/ctbrec/recorder/RecordingManager.java b/common/src/main/java/ctbrec/recorder/RecordingManager.java index fc89380e..b75f654b 100644 --- a/common/src/main/java/ctbrec/recorder/RecordingManager.java +++ b/common/src/main/java/ctbrec/recorder/RecordingManager.java @@ -122,6 +122,9 @@ public class RecordingManager { } public void delete(Recording recording) throws IOException { + int idx = recordings.indexOf(recording); + recording = recordings.get(idx); + recording.setStatus(State.DELETING); File recordingsDir = new File(config.getSettings().recordingsDir); File path = new File(recordingsDir, recording.getPath()); diff --git a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java index 8fe3b2bb..0595aa3b 100644 --- a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java +++ b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java @@ -47,6 +47,7 @@ public class RemoteRecorder implements Recorder { private JsonAdapter modelListResponseAdapter = moshi.adapter(ModelListResponse.class); private JsonAdapter recordingListResponseAdapter = moshi.adapter(RecordingListResponse.class); private JsonAdapter modelRequestAdapter = moshi.adapter(ModelRequest.class); + private JsonAdapter recordingRequestAdapter = moshi.adapter(RecordingRequest.class); private List models = Collections.emptyList(); private List onlineModels = Collections.emptyList(); @@ -335,18 +336,19 @@ public class RemoteRecorder implements Recorder { @Override public void delete(Recording recording) throws IOException, InvalidKeyException, NoSuchAlgorithmException, IllegalStateException { - String msg = "{\"action\": \"delete\", \"recording\": \""+recording.getPath()+"\"}"; + RecordingRequest recReq = new RecordingRequest("delete", recording); + String msg = recordingRequestAdapter.toJson(recReq); RequestBody body = RequestBody.create(JSON, msg); Request.Builder builder = new Request.Builder() .url("http://" + config.getSettings().httpServer + ":" + config.getSettings().httpPort + "/rec") .post(body); addHmacIfNeeded(msg, builder); Request request = builder.build(); - try(Response response = client.execute(request)) { + try (Response response = client.execute(request)) { String json = response.body().string(); RecordingListResponse resp = recordingListResponseAdapter.fromJson(json); - if(response.isSuccessful()) { - if(!resp.status.equals("success")) { + if (response.isSuccessful()) { + if (!resp.status.equals("success")) { throw new IOException("Couldn't delete recording: " + resp.msg); } else { recordings.remove(recording); @@ -384,6 +386,33 @@ public class RemoteRecorder implements Recorder { } } + public static class RecordingRequest { + private String action; + private Recording recording; + + public RecordingRequest(String action, Recording recording) { + super(); + this.action = action; + this.recording = recording; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public Recording getRecording() { + return recording; + } + + public void setRecording(Recording recording) { + this.recording = recording; + } + } + @Override public void switchStreamSource(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException, IllegalStateException { sendRequest("switch", model); diff --git a/common/src/main/java/ctbrec/recorder/download/Download.java b/common/src/main/java/ctbrec/recorder/download/Download.java index 074d903c..30dd64fc 100644 --- a/common/src/main/java/ctbrec/recorder/download/Download.java +++ b/common/src/main/java/ctbrec/recorder/download/Download.java @@ -2,6 +2,7 @@ package ctbrec.recorder.download; import java.io.File; import java.io.IOException; +import java.time.Duration; import java.time.Instant; import ctbrec.Config; @@ -14,6 +15,7 @@ public interface Download { public void stop(); public Model getModel(); public Instant getStartTime(); + public Duration getLength(); public void postprocess(Recording recording); /** diff --git a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java index 8a9500cd..38b28a9a 100644 --- a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java @@ -1,9 +1,8 @@ package ctbrec.recorder.download; -import static ctbrec.Recording.State.*; - import java.io.EOFException; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -15,14 +14,12 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.text.DecimalFormat; import java.text.NumberFormat; -import java.text.SimpleDateFormat; import java.time.Duration; import java.time.Instant; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.Date; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -30,15 +27,21 @@ import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.iheartradio.m3u8.Encoding; +import com.iheartradio.m3u8.Format; import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.ParsingMode; import com.iheartradio.m3u8.PlaylistError; import com.iheartradio.m3u8.PlaylistException; +import com.iheartradio.m3u8.PlaylistParser; +import com.iheartradio.m3u8.data.MediaPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.TrackData; import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; -import ctbrec.event.EventBusHolder; -import ctbrec.event.RecordingStateChangedEvent; +import ctbrec.Recording.State; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.recorder.PlaylistGenerator; @@ -68,7 +71,7 @@ public class HlsDownload extends AbstractHlsDownload { super.model = model; startTime = Instant.now(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern(Config.RECORDING_DATE_FORMAT); - String startTime = formatter.format(this.startTime); + String startTime = formatter.format(ZonedDateTime.ofInstant(this.startTime, ZoneId.systemDefault())); Path modelDir = FileSystems.getDefault().getPath(config.getSettings().recordingsDir, model.getSanitizedNamed()); downloadDir = FileSystems.getDefault().getPath(modelDir.toString(), startTime); } @@ -83,10 +86,6 @@ public class HlsDownload extends AbstractHlsDownload { throw new IOException(model.getName() +"'s room is not public"); } - // let the world know, that we are recording now - RecordingStateChangedEvent evt = new RecordingStateChangedEvent(getTarget(), RECORDING, model, getStartTime()); - EventBusHolder.BUS.post(evt); - String segments = getSegmentPlaylistUrl(model); if(segments != null) { if (!Files.exists(downloadDir, LinkOption.NOFOLLOW_LINKS)) { @@ -118,7 +117,10 @@ public class HlsDownload extends AbstractHlsDownload { } // split recordings - splitRecording(lastSegmentDownload); + boolean split = splitRecording(lastSegmentDownload); + if (split) { + break; + } long wait = 0; if(lastSegmentNumber == playlist.seq) { @@ -181,7 +183,9 @@ public class HlsDownload extends AbstractHlsDownload { @Override public void postprocess(Recording recording) { + recording.setStatusWithEvent(State.GENERATING_PLAYLIST, true); generatePlaylist(recording.getAbsoluteFile()); + recording.setStatusWithEvent(State.POST_PROCESSING, true); super.postprocess(recording); } @@ -215,40 +219,16 @@ public class HlsDownload extends AbstractHlsDownload { } } - private void splitRecording(Future lastSegmentDownload) { + private boolean splitRecording(Future lastSegmentDownload) { if(config.getSettings().splitRecordings > 0) { Duration recordingDuration = Duration.between(splitRecStartTime, ZonedDateTime.now()); long seconds = recordingDuration.getSeconds(); if(seconds >= config.getSettings().splitRecordings) { - File lastTargetFile = downloadDir.toFile(); - - // switch to the next dir - SimpleDateFormat sdf = new SimpleDateFormat(Config.RECORDING_DATE_FORMAT); - super.startTime = Instant.now(); - String startTime = sdf.format(new Date()); - Path modelDir = FileSystems.getDefault().getPath(config.getSettings().recordingsDir, model.getSanitizedNamed()); - LOG.debug("Switching to {}", downloadDir); - downloadDir = FileSystems.getDefault().getPath(modelDir.toString(), startTime); - downloadDir.toFile().mkdirs(); - splitRecStartTime = ZonedDateTime.now(); - - // post-process current recording - LOG.debug("Running post-processing for {}", lastTargetFile); - Thread pp = new Thread(() -> { - if(lastSegmentDownload != null) { - // wait for last segment in this directory - try { - lastSegmentDownload.get(); - } catch (InterruptedException | ExecutionException e) { - LOG.error("Couldn't wait for last segment to arrive in this directory. Playlist might be inclomplete", e); - } - } - }); - pp.setName("Post-Processing split recording"); - pp.setPriority(Thread.MIN_PRIORITY); - pp.start(); + internalStop(); + return true; } } + return false; } @Override @@ -333,4 +313,32 @@ public class HlsDownload extends AbstractHlsDownload { String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); return relativePath; } + + @Override + public Duration getLength() { + try { + File playlist = new File(getTarget(), "playlist.m3u8"); + if (playlist.exists()) { + return Duration.ofSeconds((long) getPlaylistLength(playlist)); + } + } catch (IOException | ParseException | PlaylistException e) { + LOG.error("Couldn't determine recording length", e); + } + return Duration.ofSeconds(0); + } + + private double getPlaylistLength(File playlist) throws IOException, ParseException, PlaylistException { + if (playlist.exists()) { + PlaylistParser playlistParser = new PlaylistParser(new FileInputStream(playlist), Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist m3u = playlistParser.parse(); + MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist(); + double length = 0; + for (TrackData trackData : mediaPlaylist.getTracks()) { + length += trackData.getTrackInfo().duration; + } + return length; + } else { + throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist"); + } + } } diff --git a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java index 8d6cd53c..27c5923a 100644 --- a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java @@ -41,6 +41,7 @@ import com.iheartradio.m3u8.PlaylistException; import ctbrec.Config; import ctbrec.Hmac; import ctbrec.Model; +import ctbrec.MpegUtil; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.recorder.ProgressListener; @@ -516,4 +517,14 @@ public class MergedHlsDownload extends AbstractHlsDownload { String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); return relativePath; } + + @Override + public Duration getLength() { + try { + return Duration.ofSeconds((long) MpegUtil.getFileDuration(targetFile)); + } catch (IOException e) { + LOG.error("Couldn't determine recording length", e); + return Duration.ofSeconds(0); + } + } } diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminChunkedHttpDownload.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminChunkedHttpDownload.java deleted file mode 100644 index 440e63fe..00000000 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminChunkedHttpDownload.java +++ /dev/null @@ -1,304 +0,0 @@ -package ctbrec.sites.jasmin; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URLEncoder; -import java.nio.file.Files; -import java.time.Instant; -import java.util.Random; -import java.util.regex.Pattern; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import ctbrec.Config; -import ctbrec.Model; -import ctbrec.Recording; -import ctbrec.io.HttpClient; -import ctbrec.recorder.download.Download; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.WebSocket; -import okhttp3.WebSocketListener; -import okio.ByteString; - -public class LiveJasminChunkedHttpDownload implements Download { - - private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminChunkedHttpDownload.class); - private static final transient String USER_AGENT = "Mozilla/5.0 (iPhone; CPU OS 10_14 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Mobile/14E304 Safari/605.1.15"; - - private HttpClient client; - private Model model; - private Instant startTime; - private File targetFile; - - private String applicationId; - private String sessionId; - private String jsm2SessionId; - private String sb_ip; - private String sb_hash; - private String relayHost; - private String hlsHost; - private String clientInstanceId = newClientInstanceId(); // generate a 32 digit random number - private String streamPath = "streams/clonedLiveStream"; - private boolean isAlive = true; - - public LiveJasminChunkedHttpDownload(HttpClient client) { - this.client = client; - } - - private String newClientInstanceId() { - return new java.math.BigInteger(256, new Random()).toString().substring(0, 32); - } - - @Override - public void init(Config config, Model model) { - this.model = model; - this.startTime = Instant.now(); - this.targetFile = config.getFileForRecording(model, "mp4"); - } - - @Override - public void start() throws IOException { - getPerformerDetails(model.getName()); - try { - getStreamPath(); - } catch (InterruptedException e) { - throw new IOException("Couldn't determine stream path", e); - } - - LOG.debug("appid: {}", applicationId); - LOG.debug("sessionid: {}", sessionId); - LOG.debug("jsm2sessionid: {}", jsm2SessionId); - LOG.debug("sb_ip: {}", sb_ip); - LOG.debug("sb_hash: {}", sb_hash); - LOG.debug("hls host: {}", hlsHost); - LOG.debug("clientinstanceid {}", clientInstanceId); - LOG.debug("stream path {}", streamPath); - - String rtmpUrl = "rtmp://" + sb_ip + "/" + applicationId + "?sessionId-" + sessionId + "|clientInstanceId-" + clientInstanceId; - - String m3u8 = "https://" + hlsHost + "/h5live/http/playlist.m3u8?url=" + URLEncoder.encode(rtmpUrl, "utf-8"); - m3u8 = m3u8 += "&stream=" + URLEncoder.encode(streamPath, "utf-8"); - - Request req = new Request.Builder() - .url(m3u8) - .header("User-Agent", USER_AGENT) - .header("Accept", "application/json,*/*") - .header("Accept-Language", "en") - .header("Referer", model.getUrl()) - .header("X-Requested-With", "XMLHttpRequest") - .build(); - try (Response response = client.execute(req)) { - if (response.isSuccessful()) { - System.out.println(response.body().string()); - } else { - throw new IOException(response.code() + " - " + response.message()); - } - } - - String url = "https://" + hlsHost + "/h5live/http/stream.mp4?url=" + URLEncoder.encode(rtmpUrl, "utf-8"); - url = url += "&stream=" + URLEncoder.encode(streamPath, "utf-8"); - - LOG.debug("Downloading {}", url); - req = new Request.Builder() - .url(url) - .header("User-Agent", USER_AGENT) - .header("Accept", "application/json,*/*") - .header("Accept-Language", "en") - .header("Referer", model.getUrl()) - .header("X-Requested-With", "XMLHttpRequest") - .build(); - try (Response response = client.execute(req)) { - if (response.isSuccessful()) { - FileOutputStream fos = null; - try { - Files.createDirectories(targetFile.getParentFile().toPath()); - fos = new FileOutputStream(targetFile); - - InputStream in = response.body().byteStream(); - byte[] b = new byte[10240]; - int len = -1; - while (isAlive && (len = in.read(b)) >= 0) { - fos.write(b, 0, len); - } - } catch (IOException e) { - LOG.error("Couldn't create video file", e); - } finally { - isAlive = false; - if(fos != null) { - fos.close(); - } - } - } else { - throw new IOException(response.code() + " - " + response.message()); - } - } - } - - private void getStreamPath() throws InterruptedException { - Object lock = new Object(); - - Request request = new Request.Builder() - .url("https://" + relayHost + "/?random=" + newClientInstanceId()) - .header("Origin", LiveJasmin.baseUrl) - .header("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0") - .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - .header("Accept-Language", "de,en-US;q=0.7,en;q=0.3") - .build(); - client.newWebSocket(request, new WebSocketListener() { - @Override - public void onOpen(WebSocket webSocket, Response response) { - LOG.debug("relay open {}", model.getName()); - webSocket.send("{\"event\":\"register\",\"applicationId\":\"" + applicationId - + "\",\"connectionData\":{\"jasmin2App\":true,\"isMobileClient\":false,\"platform\":\"desktop\",\"chatID\":\"freechat\"," - + "\"sessionID\":\"" + sessionId + "\"," + "\"jsm2SessionId\":\"" + jsm2SessionId + "\",\"userType\":\"user\"," + "\"performerId\":\"" - + model - + "\",\"clientRevision\":\"\",\"proxyIP\":\"\",\"playerVer\":\"nanoPlayerVersion: 3.10.3 appCodeName: Mozilla appName: Netscape appVersion: 5.0 (X11) platform: Linux x86_64\",\"livejasminTvmember\":false,\"newApplet\":true,\"livefeedtype\":null,\"gravityCookieId\":\"\",\"passparam\":\"\",\"brandID\":\"jasmin\",\"cobrandId\":\"\",\"subbrand\":\"livejasmin\",\"siteName\":\"LiveJasmin\",\"siteUrl\":\""+LiveJasmin.baseUrl+"\"," - + "\"clientInstanceId\":\"" + clientInstanceId + "\",\"armaVersion\":\"34.10.0\",\"isPassive\":false}}"); - response.close(); - } - - @Override - public void onMessage(WebSocket webSocket, String text) { - LOG.debug("relay <-- {} T{}", model.getName(), text); - JSONObject event = new JSONObject(text); - if (event.optString("event").equals("accept")) { - webSocket.send("{\"event\":\"connectSharedObject\",\"name\":\"data/chat_so\"}"); - } else if (event.optString("event").equals("updateSharedObject")) { - JSONArray list = event.getJSONArray("list"); - for (int i = 0; i < list.length(); i++) { - JSONObject obj = list.getJSONObject(i); - if (obj.optString("name").equals("streamList")) { - LOG.debug(obj.toString(2)); - streamPath = getStreamPath(obj.getJSONObject("newValue")); - LOG.debug("Stream Path: {}", streamPath); - webSocket.send("{\"event\":\"call\",\"funcName\":\"makeActive\",\"data\":[]}"); - webSocket.close(1000, ""); - synchronized (lock) { - lock.notify(); - } - } - } - }else if(event.optString("event").equals("call")) { - String func = event.optString("funcName"); - if(func.equals("closeConnection")) { - stop(); - } - } - } - - private String getStreamPath(JSONObject obj) { - String streamName = "streams/clonedLiveStream"; - int height = 0; - if(obj.has("streams")) { - JSONArray streams = obj.getJSONArray("streams"); - for (int i = 0; i < streams.length(); i++) { - JSONObject stream = streams.getJSONObject(i); - int h = stream.optInt("height"); - if(h > height) { - height = h; - streamName = stream.getString("streamNameWithFolder"); - streamName = "free/" + stream.getString("name"); - } - } - } - return streamName; - } - - @Override - public void onMessage(WebSocket webSocket, ByteString bytes) { - LOG.debug("relay <-- {} B{}", model.getName(), bytes.toString()); - } - - @Override - public void onClosed(WebSocket webSocket, int code, String reason) { - LOG.debug("relay closed {} {} {}", code, reason, model.getName()); - } - - @Override - public void onFailure(WebSocket webSocket, Throwable t, Response response) { - LOG.debug("relay failure {}", model.getName(), t); - if (response != null) { - response.close(); - } - } - }); - - synchronized (lock) { - lock.wait(); - } - } - - protected void getPerformerDetails(String name) throws IOException { - String url = "https://m."+LiveJasmin.baseDomain+"/en/chat-html5/" + name; - Request req = new Request.Builder() - .url(url) - .header("User-Agent", USER_AGENT) - .header("Accept", "application/json,*/*") - .header("Accept-Language", "en") - .header("Referer", LiveJasmin.baseUrl) - .header("X-Requested-With", "XMLHttpRequest") - .build(); - try (Response response = client.execute(req)) { - if (response.isSuccessful()) { - String body = response.body().string(); - JSONObject json = new JSONObject(body); - // System.out.println(json.toString(2)); - if (json.optBoolean("success")) { - JSONObject data = json.getJSONObject("data"); - JSONObject config = data.getJSONObject("config"); - JSONObject armageddonConfig = config.getJSONObject("armageddonConfig"); - JSONObject chatRoom = config.getJSONObject("chatRoom"); - sessionId = armageddonConfig.getString("sessionid"); - jsm2SessionId = armageddonConfig.getString("jsm2session"); - sb_hash = chatRoom.getString("sb_hash"); - sb_ip = chatRoom.getString("sb_ip"); - applicationId = "memberChat/jasmin" + name + sb_hash; - hlsHost = "dss-hls-" + sb_ip.replace('.', '-') + ".dditscdn.com"; - relayHost = "dss-relay-" + sb_ip.replace('.', '-') + ".dditscdn.com"; - } else { - throw new IOException("Response was not successful: " + body); - } - } else { - throw new IOException(response.code() + " - " + response.message()); - } - } - } - - @Override - public void stop() { - isAlive = false; - } - - @Override - public File getTarget() { - return targetFile; - } - - @Override - public Model getModel() { - return model; - } - - @Override - public Instant getStartTime() { - return startTime; - } - - @Override - public void postprocess(Recording recording) { - } - - @Override - public String getPath(Model model) { - String absolutePath = targetFile.getAbsolutePath(); - String recordingsDir = Config.getInstance().getSettings().recordingsDir; - String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); - return relativePath; - } -} diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminWebSocketDownload.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminWebSocketDownload.java deleted file mode 100644 index 51790670..00000000 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminWebSocketDownload.java +++ /dev/null @@ -1,368 +0,0 @@ -package ctbrec.sites.jasmin; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.file.Files; -import java.time.Instant; -import java.util.regex.Pattern; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.eventbus.Subscribe; - -import ctbrec.Config; -import ctbrec.Model; -import ctbrec.Recording; -import ctbrec.event.Event; -import ctbrec.event.EventBusHolder; -import ctbrec.event.ModelStateChangedEvent; -import ctbrec.io.HttpClient; -import ctbrec.recorder.download.Download; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.WebSocket; -import okhttp3.WebSocketListener; -import okio.ByteString; - -public class LiveJasminWebSocketDownload implements Download { - private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminWebSocketDownload.class); - - private String applicationId; - private String sessionId; - private String jsm2SessionId; - private String sb_ip; - private String sb_hash; - private String relayHost; - private String streamHost; - private String clientInstanceId = "01234567890123456789012345678901"; // TODO where to get or generate a random id? - private String streamPath = "streams/clonedLiveStream"; - private WebSocket relay; - private WebSocket stream; - - protected boolean connectionClosed; - - private HttpClient client; - private Model model; - private Instant startTime; - private File targetFile; - - public LiveJasminWebSocketDownload(HttpClient client) { - this.client = client; - } - - @Override - public void init(Config config, Model model) { - this.model = model; - this.startTime = Instant.now(); - this.targetFile = config.getFileForRecording(model, "mp4"); - } - - @Override - public void start() throws IOException { - getPerformerDetails(model.getName()); - LOG.debug("appid: {}", applicationId); - LOG.debug("sessionid: {}",sessionId); - LOG.debug("jsm2sessionid: {}",jsm2SessionId); - LOG.debug("sb_ip: {}",sb_ip); - LOG.debug("sb_hash: {}",sb_hash); - LOG.debug("relay host: {}",relayHost); - LOG.debug("stream host: {}",streamHost); - LOG.debug("clientinstanceid {}",clientInstanceId); - - EventBusHolder.BUS.register(this); - - Request request = new Request.Builder() - .url("https://" + relayHost + "/") - .header("Origin", LiveJasmin.baseUrl) - .header("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0") - .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - .header("Accept-Language", "de,en-US;q=0.7,en;q=0.3") - .build(); - relay = client.newWebSocket(request, new WebSocketListener() { - boolean streamSocketStarted = false; - - @Override - public void onOpen(WebSocket webSocket, Response response) { - LOG.trace("relay open {}", model.getName()); - sendToRelay("{\"event\":\"register\",\"applicationId\":\"" + applicationId - + "\",\"connectionData\":{\"jasmin2App\":true,\"isMobileClient\":false,\"platform\":\"desktop\",\"chatID\":\"freechat\"," - + "\"sessionID\":\"" + sessionId + "\"," + "\"jsm2SessionId\":\"" + jsm2SessionId + "\",\"userType\":\"user\"," + "\"performerId\":\"" - + model - + "\",\"clientRevision\":\"\",\"proxyIP\":\"\",\"playerVer\":\"nanoPlayerVersion: 3.10.3 appCodeName: Mozilla appName: Netscape appVersion: 5.0 (X11) platform: Linux x86_64\",\"livejasminTvmember\":false,\"newApplet\":true,\"livefeedtype\":null,\"gravityCookieId\":\"\",\"passparam\":\"\",\"brandID\":\"jasmin\",\"cobrandId\":\"\",\"subbrand\":\"livejasmin\",\"siteName\":\"LiveJasmin\",\"siteUrl\":\""+LiveJasmin.baseUrl+"\"," - + "\"clientInstanceId\":\"" + clientInstanceId + "\",\"armaVersion\":\"34.10.0\",\"isPassive\":false}}"); - response.close(); - } - - @Override - public void onMessage(WebSocket webSocket, String text) { - LOG.trace("relay <-- {} T{}", model.getName(), text); - JSONObject event = new JSONObject(text); - if (event.optString("event").equals("accept")) { - new Thread(() -> { - sendToRelay("{\"event\":\"connectSharedObject\",\"name\":\"data/chat_so\"}"); - }).start(); - } else if (event.optString("event").equals("updateSharedObject")) { - JSONArray list = event.getJSONArray("list"); - for (int i = 0; i < list.length(); i++) { - JSONObject obj = list.getJSONObject(i); - if (obj.optString("name").equals("streamList")) { - //LOG.debug(obj.toString(2)); - streamPath = getStreamPath(obj.getJSONObject("newValue")); - } else if(obj.optString("name").equals("isPrivate") - || obj.optString("name").equals("onPrivate") - || obj.optString("name").equals("onPrivateAll") - || obj.optString("name").equals("onPrivateLJ")) - { - if(obj.optBoolean("newValue")) { - // model went private, stop recording - LOG.debug("Model {} state changed to private -> stopping download", model.getName()); - stop(); - } - } else if(obj.optString("name").equals("recommendedBandwidth") || obj.optString("name").equals("realQualityData")) { - // stream quality related -> do nothing - } else { - LOG.debug("{} -{}", model.getName(), obj.toString()); - } - } - - if (!streamSocketStarted) { - streamSocketStarted = true; - sendToRelay("{\"event\":\"call\",\"funcName\":\"makeActive\",\"data\":[]}"); - new Thread(() -> { - try { - startStreamSocket(); - } catch (Exception e) { - LOG.error("Couldn't start stream websocket", e); - stop(); - } - }).start(); - } - } else if(event.optString("event").equals("call")) { - String func = event.optString("funcName"); - if (func.equals("closeConnection")) { - connectionClosed = true; - // System.out.println(event.get("data")); - stop(); - } else if (func.equals("addLine")) { - // chat message -> ignore - } else if (func.equals("receiveInvitation")) { - // invitation to private show -> ignore - } else { - LOG.debug("{} -{}", model.getName(), event.toString()); - } - } else { - if(!event.optString("event").equals("pong")) - LOG.debug("{} -{}", model.getName(), event.toString()); - } - } - - private String getStreamPath(JSONObject obj) { - String streamName = "streams/clonedLiveStream"; - int height = 0; - if(obj.has("streams")) { - JSONArray streams = obj.getJSONArray("streams"); - for (int i = 0; i < streams.length(); i++) { - JSONObject stream = streams.getJSONObject(i); - int h = stream.optInt("height"); - if(h > height) { - height = h; - streamName = stream.getString("streamNameWithFolder"); - streamName = "free/" + stream.getString("name"); - } - } - } - return streamName; - } - - @Override - public void onMessage(WebSocket webSocket, ByteString bytes) { - LOG.trace("relay <-- {} B{}", model.getName(), bytes.toString()); - } - - @Override - public void onClosed(WebSocket webSocket, int code, String reason) { - LOG.trace("relay closed {} {} {}", code, reason, model.getName()); - stop(); - } - - @Override - public void onFailure(WebSocket webSocket, Throwable t, Response response) { - if(!connectionClosed) { - LOG.trace("relay failure {}", model.getName(), t); - stop(); - if (response != null) { - response.close(); - } - } - } - }); - } - - @Subscribe - public void handleEvent(Event evt) { - if(evt.getType() == Event.Type.MODEL_STATUS_CHANGED) { - ModelStateChangedEvent me = (ModelStateChangedEvent) evt; - if(me.getModel().equals(model) && me.getOldState() == Model.State.ONLINE) { - LOG.debug("Model {} state changed to {} -> stopping download", me.getNewState(), model.getName()); - stop(); - } - } - } - - private void sendToRelay(String msg) { - LOG.trace("relay --> {} {}", model.getName(), msg); - relay.send(msg); - } - - protected void getPerformerDetails(String name) throws IOException { - String url = "https://m." + LiveJasmin.baseDomain + "/en/chat-html5/" + name; - Request req = new Request.Builder() - .url(url) - .header("User-Agent", "Mozilla/5.0 (iPhone; CPU OS 10_14 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Mobile/14E304 Safari/605.1.15") - .header("Accept", "application/json,*/*") - .header("Accept-Language", "en") - .header("Referer", LiveJasmin.baseUrl) - .header("X-Requested-With", "XMLHttpRequest") - .build(); - try (Response response = client.execute(req)) { - if (response.isSuccessful()) { - String body = response.body().string(); - JSONObject json = new JSONObject(body); - // System.out.println(json.toString(2)); - if (json.optBoolean("success")) { - JSONObject data = json.getJSONObject("data"); - JSONObject config = data.getJSONObject("config"); - JSONObject armageddonConfig = config.getJSONObject("armageddonConfig"); - JSONObject chatRoom = config.getJSONObject("chatRoom"); - sessionId = armageddonConfig.getString("sessionid"); - jsm2SessionId = armageddonConfig.getString("jsm2session"); - sb_hash = chatRoom.getString("sb_hash"); - sb_ip = chatRoom.getString("sb_ip"); - applicationId = "memberChat/jasmin" + name + sb_hash; - relayHost = "dss-relay-" + sb_ip.replace('.', '-') + ".dditscdn.com"; - streamHost = "dss-live-" + sb_ip.replace('.', '-') + ".dditscdn.com"; - } else { - throw new IOException("Response was not successful: " + body); - } - } else { - throw new IOException(response.code() + " - " + response.message()); - } - } - } - - private void startStreamSocket() throws UnsupportedEncodingException { - String rtmpUrl = "rtmp://" + sb_ip + "/" + applicationId + "?sessionId-" + sessionId + "|clientInstanceId-" + clientInstanceId; - String url = "https://" + streamHost + "/stream/?url=" + URLEncoder.encode(rtmpUrl, "utf-8"); - url = url += "&stream=" + URLEncoder.encode(streamPath, "utf-8") + "&cid=863621&pid=49247581854"; - LOG.trace(rtmpUrl); - LOG.trace(url); - - Request request = new Request.Builder() - .url(url) - .header("Origin", LiveJasmin.baseUrl) - .header("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0") - .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8").header("Accept-Language", "de,en-US;q=0.7,en;q=0.3") - .build(); - stream = client.newWebSocket(request, new WebSocketListener() { - FileOutputStream fos; - - @Override - public void onOpen(WebSocket webSocket, Response response) { - LOG.trace("stream open {}", model.getName()); - // webSocket.send("{\"event\":\"ping\"}"); - // webSocket.send(""); - response.close(); - try { - Files.createDirectories(targetFile.getParentFile().toPath()); - fos = new FileOutputStream(targetFile); - } catch (IOException e) { - LOG.error("Couldn't create video file", e); - stop(); - } - } - - @Override - public void onMessage(WebSocket webSocket, String text) { - LOG.trace("stream <-- {} T{}", model.getName(), text); - JSONObject event = new JSONObject(text); - if(event.optString("eventType").equals("onRandomAccessPoint")) { - // send ping - sendToRelay("{\"event\":\"ping\"}"); - } - } - - @Override - public void onMessage(WebSocket webSocket, ByteString bytes) { - //System.out.println("stream <-- B" + bytes.toString()); - try { - fos.write(bytes.toByteArray()); - } catch (IOException e) { - LOG.error("Couldn't write video chunk to file", e); - stop(); - } - } - - @Override - public void onClosed(WebSocket webSocket, int code, String reason) { - LOG.trace("stream closed {} {} {}", code, reason, model.getName()); - stop(); - } - - @Override - public void onFailure(WebSocket webSocket, Throwable t, Response response) { - if(!connectionClosed) { - LOG.trace("stream failure {}", model.getName(), t); - stop(); - if (response != null) { - response.close(); - } - } - } - }); - } - - @Override - public void stop() { - connectionClosed = true; - EventBusHolder.BUS.unregister(this); - if (stream != null) { - stream.close(1000, ""); - } - if (relay != null) { - relay.close(1000, ""); - } - } - - @Override - public File getTarget() { - return targetFile; - } - - @Override - public Model getModel() { - return model; - } - - @Override - public Instant getStartTime() { - return startTime; - } - - @Override - public void postprocess(Recording recording) { - } - - @Override - public String getPath(Model model) { - String absolutePath = targetFile.getAbsolutePath(); - String recordingsDir = Config.getInstance().getSettings().recordingsDir; - String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); - return relativePath; - } -} diff --git a/server/src/main/java/ctbrec/recorder/server/HttpServer.java b/server/src/main/java/ctbrec/recorder/server/HttpServer.java index d2bb6d13..ff270d1b 100644 --- a/server/src/main/java/ctbrec/recorder/server/HttpServer.java +++ b/server/src/main/java/ctbrec/recorder/server/HttpServer.java @@ -75,8 +75,8 @@ public class HttpServer { site.init(); } } - OnlineMonitor monitor = new OnlineMonitor(recorder); - monitor.start(); + onlineMonitor = new OnlineMonitor(recorder); + onlineMonitor.start(); startHttpServer(); } diff --git a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java index e75efc05..c2aca727 100644 --- a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java @@ -120,13 +120,10 @@ public class RecorderServlet extends AbstractCtbrecServlet { resp.getWriter().write("]}"); break; case "delete": - String path = request.recording; - Recording rec = new Recording(); - rec.setPath(path); - recorder.delete(rec); + recorder.delete(request.recording); recAdapter = moshi.adapter(Recording.class); resp.getWriter().write("{\"status\": \"success\", \"msg\": \"List of recordings\", \"recordings\": ["); - resp.getWriter().write(recAdapter.toJson(rec)); + resp.getWriter().write(recAdapter.toJson(request.recording)); resp.getWriter().write("]}"); break; case "switch": @@ -178,6 +175,6 @@ public class RecorderServlet extends AbstractCtbrecServlet { private static class Request { public String action; public Model model; - public String recording; + public Recording recording; } }