diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index 200983dd..3bfe297b 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -38,7 +38,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; @Slf4j public abstract class HttpClient { @Getter - private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(10, 2, TimeUnit.MINUTES); + private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(256, 2, TimeUnit.MINUTES); @Getter protected CookieJarImpl cookieJar; diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index c09f5ab7..fb5b1914 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -302,7 +302,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { throw new HttpException(response.code(), response.message()); } } catch (SocketTimeoutException e) { - LOG.debug("Playlist request timed out ({}ms) for model {} {} time{}", config.getSettings().playlistRequestTimeout, model, + LOG.debug("Playlist request timed out ({}ms) for model {}:{} {} time{}", config.getSettings().playlistRequestTimeout, model.getSite().getName(), model, ++consecutivePlaylistTimeouts, (consecutivePlaylistTimeouts > 1) ? 's' : ""); // times out, return an empty playlist, so that the process can continue without wasting much more time recordingEvents.add(RecordingEvent.of("Playlist request timed out " + consecutivePlaylistTimeouts)); @@ -310,7 +310,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { } catch (Exception e) { consecutivePlaylistErrors++; throw e; - } + } } private SegmentPlaylist parsePlaylist(String segmentPlaylistUrl, InputStream inputStream) throws IOException, ParseException, PlaylistException { diff --git a/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java index 74d0bb93..a7a30066 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java @@ -12,10 +12,13 @@ import okhttp3.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.datatype.jsr310.deser.DurationDeserializer; + import javax.crypto.NoSuchPaddingException; import java.io.*; import java.net.MalformedURLException; import java.net.URL; +import java.net.URI; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -23,6 +26,8 @@ import java.util.HashMap; import java.util.Objects; import java.util.Optional; import java.util.concurrent.Callable; +import java.time.Instant; +import java.time.Duration; import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL; import static ctbrec.recorder.download.hls.AbstractHlsDownload.addHeaders; @@ -36,6 +41,7 @@ public class SegmentDownload implements Callable { protected final Segment segment; protected final Model model; protected final OutputStream out; + protected final Instant createdAt; private long size = 0; protected boolean failed = false; @@ -48,12 +54,23 @@ public class SegmentDownload implements Callable { this.segment = segment; this.client = client; this.out = out; - this.url = new URL(segment.url); + this.url = URI.create(segment.url).toURL(); + this.createdAt = Instant.now(); } @Override public SegmentDownload call() { + var expiresAt = createdAt.plusSeconds(10); + for (int tries = 1; tries <= 3 && !Thread.currentThread().isInterrupted(); tries++) { // NOSONAR + if (expiresAt.isBefore(Instant.now())) { + // segment has sexpired, skip it + LOG.warn("Segment for model {} is late {} seconds", model, Duration.between(expiresAt, Instant.now())); + failed = true; + exception = new Exception("Segment expired"); + break; + } + Request request = createRequest(); try (Response response = client.execute(request)) { handleResponse(response);