ppbbww

commit a125eb996cc8a411492fc8c7d2ce96dafa37acef
parent 64f208a8683d5c659039dbe3ba98b0e5be88c342
Author: david cochran <about.trout@gmail.com>
Date:   Sat, 17 Feb 2024 19:23:13 -0800

read cam names from command line args

Diffstat:
M
README.md
 | 
2
+-
M
ppboatwatch/sample_streams.py
 | 
23
+++++++++++++++++++----
M
ppboatwatch/stream_sampler.py
 | 
13
++++++++++++-

3 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/README.md b/README.md
@@ -17,7 +17,7 @@ $ pip install -e .
 #### Accumulate data by sampling streams. 
 
 ```
-$ sample-streams 
+$ sample-streams --cams mavericks mavericksov
 2024-02-11 20:36:15,997 cam_name=mavericksov num_frames=4 Extracted frames
 2024-02-11 20:36:15,998 cam_name=mavericksov delay=12 Sleeping
 2024-02-11 20:36:16,583 cam_name=mavericks num_frames=5 Extracted frames
diff --git a/ppboatwatch/sample_streams.py b/ppboatwatch/sample_streams.py
@@ -1,3 +1,4 @@
+import argparse
 import asyncio
 import logging
 import random
@@ -10,7 +11,9 @@ async def sample_stream(cam_name, data_dir):
     while True:
         try:
             frames = await ss.get_recent_frames()
-            logging.info(f"cam_name={cam_name} num_frames={len(frames)} Extracted frames")
+            logging.info(
+                f"cam_name={cam_name} num_frames={len(frames)} Extracted frames"
+            )
         except Exception as ex:
             logging.error(f"cam_name={cam_name} Failed to get_recent_frames: {ex}")
         # TODO: Decrease delay but only keep 1 frame?
@@ -26,7 +29,19 @@ async def main_task(cams, data_dir):
 
 
 def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--cams",
+        nargs="+",
+        action="extend",
+        help="Surfline cam(s) to crawl",
+        required=True,
+    )
+    parser.add_argument(
+        "-d", "--dir", default="data", help="Directory to store extracted frames"
+    )
+    args = parser.parse_args()
+
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
-    cams = ["mavericks", "mavericksov"]  # TODO: argparse.
-    data_dir = "./data"                  # TODO: argparse
-    asyncio.run(main_task(cams, data_dir))
+    asyncio.run(main_task(args.cams, args.dir))
diff --git a/ppboatwatch/stream_sampler.py b/ppboatwatch/stream_sampler.py
@@ -8,6 +8,13 @@ import subprocess
 import tempfile
 import time
 
+DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0",
+    "Origin": "https://www.surfline.com",
+    "Referer": "https://www.surfline.com/",
+    "DNT": "1",
+}
+
 
 class StreamSampler:
     def __init__(self, cam_name, data_dir="./data"):
@@ -16,7 +23,11 @@ class StreamSampler:
         self.data_dir = data_dir
 
     async def get_recent_frames(self):
-        async with aiohttp.ClientSession(raise_for_status=True) as session:
+        async with aiohttp.ClientSession(
+            raise_for_status=True,
+            skip_auto_headers=list(DEFAULT_HEADERS.keys()),
+            headers=DEFAULT_HEADERS,
+        ) as session:
             chunk_url = await self.__get_recent_chunk_url(session)
             return self.__extract_frames(chunk_url)