Fix AmateurTV downloads. They switched from HLS segments to a MP4 stream

common/src/main/java/ctbrec/sites/amateurtv/AmateurTvDownload.java

@@ -0,0 +1,170 @@
+package ctbrec.sites.amateurtv;
+
+import ctbrec.Config;
+import ctbrec.Model;
+import ctbrec.Recording;
+import ctbrec.io.BandwidthMeter;
+import ctbrec.io.HttpClient;
+import ctbrec.io.HttpException;
+import ctbrec.recorder.download.AbstractDownload;
+import ctbrec.recorder.download.Download;
+import ctbrec.recorder.download.StreamSource;
+import okhttp3.Request;
+import okhttp3.Response;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.regex.Pattern;
+
+import static ctbrec.io.HttpConstants.*;
+
+public class AmateurTvDownload extends AbstractDownload {
+
+        private static final Logger LOG = LoggerFactory.getLogger(AmateurTvDownload.class);
+        private static final int MAX_SECONDS_WITHOUT_TRANSFER = 20;
+
+        private final HttpClient httpClient;
+        private FileOutputStream fout;
+        private Instant timeOfLastTransfer = Instant.MAX;
+
+        private volatile boolean running;
+        private volatile boolean started;
+
+        private File targetFile;
+
+        public AmateurTvDownload(HttpClient httpClient) {
+            this.httpClient = httpClient;
+        }
+
+        @Override
+        public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException {
+            this.config = config;
+            this.model = model;
+            this.startTime = startTime;
+            this.downloadExecutor = executorService;
+            splittingStrategy = initSplittingStrategy(config.getSettings());
+            targetFile = config.getFileForRecording(model, "mp4", startTime);
+            timeOfLastTransfer = Instant.now();
+        }
+
+        @Override
+        public void stop() {
+            running = false;
+        }
+
+        @Override
+        public void finalizeDownload() {
+            if (fout != null) {
+                try {
+                    LOG.debug("Closing recording file {}", targetFile);
+                    fout.close();
+                } catch (IOException e) {
+                    LOG.error("Error while closing recording file {}", targetFile, e);
+                }
+            }
+        }
+
+        @Override
+        public boolean isRunning() {
+            return running;
+        }
+
+        @Override
+        public void postprocess(Recording recording) {
+            // nothing to do
+        }
+
+        @Override
+        public File getTarget() {
+            return targetFile;
+        }
+
+        @Override
+        public String getPath(Model model) {
+            String absolutePath = targetFile.getAbsolutePath();
+            String recordingsDir = Config.getInstance().getSettings().recordingsDir;
+            String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), "");
+            return relativePath;
+        }
+
+        @Override
+        public boolean isSingleFile() {
+            return true;
+        }
+
+        @Override
+        public long getSizeInByte() {
+            return getTarget().length();
+        }
+
+        @Override
+        public Download call() throws Exception {
+            if (!started) {
+                started = true;
+                startDownload();
+            }
+
+            if (splittingStrategy.splitNecessary(this)) {
+                stop();
+                rescheduleTime = Instant.now();
+            } else {
+                rescheduleTime = Instant.now().plusSeconds(5);
+            }
+            if (!model.isOnline(true)) {
+                stop();
+            }
+            if (Duration.between(timeOfLastTransfer, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) {
+                LOG.info("No video data received for {} seconds. Stopping recording for model {}", MAX_SECONDS_WITHOUT_TRANSFER, model);
+                stop();
+            }
+            return this;
+        }
+
+    private void startDownload() {
+        downloadExecutor.submit(() -> {
+            running = true;
+            try {
+                StreamSource src = model.getStreamSources().get(0);
+                LOG.debug("Loading video from {}", src.mediaPlaylistUrl);
+                Request request = new Request.Builder()
+                        .url(src.mediaPlaylistUrl)
+                        .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                        .header(ACCEPT, "*/*")
+                        .header(ACCEPT_LANGUAGE, "en")
+                        .header(ORIGIN, AmateurTv.baseUrl)
+                        .build();
+                try (Response resp = httpClient.execute(request)) {
+                    if (resp.isSuccessful()) {
+                        LOG.debug("Recording video stream to {}", targetFile);
+                        Files.createDirectories(targetFile.getParentFile().toPath());
+                        fout = new FileOutputStream(targetFile);
+
+                        InputStream in = Objects.requireNonNull(resp.body()).byteStream();
+                        byte[] b = new byte[1024];
+                        int len;
+                        while (running && !Thread.currentThread().isInterrupted() && (len = in.read(b)) >= 0) {
+                            fout.write(b, 0, len);
+                            timeOfLastTransfer = Instant.now();
+                            BandwidthMeter.add(len);
+                        }
+                    } else {
+                        throw new HttpException(resp.code(), resp.message());
+                    }
+                }
+            } catch (Exception e) {
+               LOG.error("Error while downloading MP4", e);
+            }
+            running = false;
+        });
+    }
+
+}

common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java

@@ -1,8 +1,23 @@
 package ctbrec.sites.amateurtv;
 
-import static ctbrec.Model.State.*;
+import com.iheartradio.m3u8.*;
-import static ctbrec.io.HttpConstants.*;
+import com.iheartradio.m3u8.data.MediaPlaylist;
+import com.iheartradio.m3u8.data.Playlist;
+import ctbrec.AbstractModel;
+import ctbrec.Config;
+import ctbrec.io.HttpClient;
+import ctbrec.io.HttpException;
+import ctbrec.recorder.download.Download;
+import ctbrec.recorder.download.StreamSource;
+import okhttp3.FormBody;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import javax.xml.bind.JAXBException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -11,33 +26,8 @@ import java.util.Locale;
 import java.util.Objects;
 import java.util.concurrent.ExecutionException;
 
-import javax.xml.bind.JAXBException;
+import static ctbrec.Model.State.*;
-
+import static ctbrec.io.HttpConstants.*;
-import org.json.JSONObject;
-import org.jsoup.nodes.Element;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.iheartradio.m3u8.Encoding;
-import com.iheartradio.m3u8.Format;
-import com.iheartradio.m3u8.ParseException;
-import com.iheartradio.m3u8.ParsingMode;
-import com.iheartradio.m3u8.PlaylistException;
-import com.iheartradio.m3u8.PlaylistParser;
-import com.iheartradio.m3u8.data.MasterPlaylist;
-import com.iheartradio.m3u8.data.Playlist;
-import com.iheartradio.m3u8.data.PlaylistData;
-import com.iheartradio.m3u8.data.StreamInfo;
-
-import ctbrec.AbstractModel;
-import ctbrec.Config;
-import ctbrec.io.HttpClient;
-import ctbrec.io.HttpException;
-import ctbrec.recorder.download.StreamSource;
-import okhttp3.FormBody;
-import okhttp3.Request;
-import okhttp3.RequestBody;
-import okhttp3.Response;
 
 public class AmateurTvModel extends AbstractModel {
 
@@ -79,28 +69,17 @@ public class AmateurTvModel extends AbstractModel {
         Request req = new Request.Builder().url(streamUrl).build();
         try (Response response = site.getHttpClient().execute(req)) {
             if (response.isSuccessful()) {
-                InputStream inputStream = response.body().byteStream();
+                InputStream inputStream = Objects.requireNonNull(response.body()).byteStream();
                 PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
                 Playlist playlist = parser.parse();
-                MasterPlaylist master = playlist.getMasterPlaylist();
+                MediaPlaylist media = playlist.getMediaPlaylist();
-                for (PlaylistData playlistData : master.getPlaylists()) {
+                String baseUrl = streamUrl.substring(0, streamUrl.lastIndexOf('/') + 1);
+                String vodUri = baseUrl + media.getTracks().get(0).getUri();
                 StreamSource streamsource = new StreamSource();
-                    Element img = new Element("img");
+                streamsource.mediaPlaylistUrl = vodUri;
-                    img.setBaseUri(streamUrl);
-                    img.attr("src", playlistData.getUri());
-                    streamsource.mediaPlaylistUrl = img.absUrl("src");
-                    if (playlistData.hasStreamInfo()) {
-                        StreamInfo info = playlistData.getStreamInfo();
-                        streamsource.bandwidth = info.getBandwidth();
-                        streamsource.width = info.hasResolution() ? info.getResolution().width : 0;
-                        streamsource.height = info.hasResolution() ? info.getResolution().height : 0;
-                    } else {
-                        streamsource.bandwidth = 0;
                 streamsource.width = 0;
                 streamsource.height = 0;
-                    }
                 streamSources.add(streamsource);
-                }
             } else {
                 throw new HttpException(response.code(), response.message());
             }
@@ -111,7 +90,7 @@ public class AmateurTvModel extends AbstractModel {
     private String getStreamUrl() throws IOException {
         JSONObject json = getModelInfo();
         JSONObject videoTech = json.getJSONObject("videoTechnologies");
-        return videoTech.getString("hlsV2");
+        return videoTech.getString("fmp4-hls");
     }
 
     @Override
@@ -165,7 +144,7 @@ public class AmateurTvModel extends AbstractModel {
                 .build();
         try (Response resp = site.getHttpClient().execute(req)) {
             if (resp.isSuccessful()) {
-                String msg = resp.body().string();
+                String msg = Objects.requireNonNull(resp.body()).string();
                 JSONObject json = new JSONObject(msg);
                 if (Objects.equals(json.getString("result"), "OK")) {
                     LOG.debug("Follow/Unfollow -> {}", msg);
@@ -193,4 +172,9 @@ public class AmateurTvModel extends AbstractModel {
             return json;
         }
     }
+
+    @Override
+    public Download createDownload() {
+        return new AmateurTvDownload(getSite().getHttpClient());
+    }
 }