majjo/Twitch-Drops-Miner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix conflict, merge master with improvements-drops

Browse Source
This commit is contained in:
Alessandro Maggio
2021-02-13 11:18:48 +01:00
parent 2a097a6324 a8baad782c
commit 4fcaeae4cf
6 changed files with 203 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
TwitchChannelPointsMiner/TwitchChannelPointsMiner.py
View File

@@ -25,6 +25,7 @@ from TwitchChannelPointsMiner.utils import (
_millify,
at_least_one_value_in_settings_is,
get_user_agent,
internet_connection_available,
set_default_settings,
)
@@ -182,6 +183,7 @@ class TwitchChannelPointsMiner:
target=self.twitch.send_minute_watched_events,
args=(self.streamers, self.priority),
)
self.minute_watcher_thread.name = "Minute watcher"
self.minute_watcher_thread.start()
self.ws_pool = WebSocketsPool(
@@ -241,8 +243,9 @@ class TwitchChannelPointsMiner:
# Check if is not None because maybe we have already created a new connection on array+1 and now index is None
for index in range(0, len(self.ws_pool.ws)):
if (
self.ws_pool.ws[index] is not None
self.ws_pool.ws[index].is_reconneting is False
and self.ws_pool.ws[index].elapsed_last_ping() > 10
and internet_connection_available() is True
):
logger.info(
f"#{index} - The last PING was sent more than 10 minutes ago. Reconnecting to the WebSocket..."

186
TwitchChannelPointsMiner/classes/Twitch.py
View File

@@ -24,7 +24,7 @@ from TwitchChannelPointsMiner.classes.Exceptions import (
from TwitchChannelPointsMiner.classes.Settings import Priority, Settings
from TwitchChannelPointsMiner.classes.TwitchLogin import TwitchLogin
from TwitchChannelPointsMiner.constants import API, CLIENT_ID, GQLOperations
from TwitchChannelPointsMiner.utils import _millify
from TwitchChannelPointsMiner.utils import _millify, internet_connection_available
logger = logging.getLogger(__name__)
@@ -49,76 +49,90 @@ class Twitch(object):
def update_stream(self, streamer):
if streamer.stream.update_required() is True:
stream_info = self.get_stream_info(streamer)
streamer.stream.update(
broadcast_id=stream_info["stream"]["id"],
title=stream_info["broadcastSettings"]["title"],
game=stream_info["broadcastSettings"]["game"],
tags=stream_info["stream"]["tags"],
viewers_count=stream_info["stream"]["viewersCount"],
)
if stream_info is not None:
streamer.stream.update(
broadcast_id=stream_info["stream"]["id"],
title=stream_info["broadcastSettings"]["title"],
game=stream_info["broadcastSettings"]["game"],
tags=stream_info["stream"]["tags"],
viewers_count=stream_info["stream"]["viewersCount"],
)
event_properties = {
"channel_id": streamer.channel_id,
"broadcast_id": streamer.stream.broadcast_id,
"player": "site",
"user_id": self.twitch_login.get_user_id(),
}
event_properties = {
"channel_id": streamer.channel_id,
"broadcast_id": streamer.stream.broadcast_id,
"player": "site",
"user_id": self.twitch_login.get_user_id(),
}
if (
streamer.stream.game_name() is not None
and streamer.settings.claim_drops is True
):
event_properties["game"] = streamer.stream.game_name()
if (
streamer.stream.game_name() is not None
and streamer.settings.claim_drops is True
):
event_properties["game"] = streamer.stream.game_name()
streamer.stream.payload = [
{"event": "minute-watched", "properties": event_properties}
]
streamer.stream.payload = [
{"event": "minute-watched", "properties": event_properties}
]
def get_spade_url(self, streamer):
headers = {"User-Agent": self.user_agent}
main_page_request = requests.get(streamer.streamer_url, headers=headers)
response = main_page_request.text
settings_url = re.search(
"(https://static.twitchcdn.net/config/settings.*?js)", response
).group(1)
try:
headers = {"User-Agent": self.user_agent}
main_page_request = requests.get(streamer.streamer_url, headers=headers)
response = main_page_request.text
settings_url = re.search(
"(https://static.twitchcdn.net/config/settings.*?js)", response
).group(1)
settings_request = requests.get(settings_url, headers=headers)
response = settings_request.text
streamer.stream.spade_url = re.search('"spade_url":"(.*?)"', response).group(1)
settings_request = requests.get(settings_url, headers=headers)
response = settings_request.text
streamer.stream.spade_url = re.search(
'"spade_url":"(.*?)"', response
).group(1)
except requests.exceptions.RequestException as e:
logger.error(f"Something went wrong during extraction of 'spade_url': {e}")
def post_gql_request(self, json_data):
response = requests.post(
GQLOperations.url,
json=json_data,
headers={
"Authorization": f"OAuth {self.twitch_login.get_auth_token()}",
"Client-Id": CLIENT_ID,
"User-Agent": self.user_agent,
},
)
logger.debug(
f"Data: {json_data}, Status code: {response.status_code}, Content: {response.text}"
)
return response.json()
try:
response = requests.post(
GQLOperations.url,
json=json_data,
headers={
"Authorization": f"OAuth {self.twitch_login.get_auth_token()}",
"Client-Id": CLIENT_ID,
"User-Agent": self.user_agent,
},
)
logger.debug(
f"Data: {json_data}, Status code: {response.status_code}, Content: {response.text}"
)
return response.json()
except requests.exceptions.RequestException as e:
logger.error(
f"Error with GQLOperations ({json_data['operationName']}): {e}"
)
return {}
def get_broadcast_id(self, streamer):
json_data = copy.deepcopy(GQLOperations.WithIsStreamLiveQuery)
json_data["variables"] = {"id": streamer.channel_id}
response = self.post_gql_request(json_data)
stream = response["data"]["user"]["stream"]
if stream is not None:
return stream["id"]
else:
raise StreamerIsOfflineException
if response != {}:
stream = response["data"]["user"]["stream"]
if stream is not None:
return stream["id"]
else:
raise StreamerIsOfflineException
def get_stream_info(self, streamer):
json_data = copy.deepcopy(GQLOperations.VideoPlayerStreamInfoOverlayChannel)
json_data["variables"] = {"channel": streamer.username}
response = self.post_gql_request(json_data)
if response["data"]["user"]["stream"] is None:
raise StreamerIsOfflineException
else:
return response["data"]["user"]
if response != {}:
if response["data"]["user"]["stream"] is None:
raise StreamerIsOfflineException
else:
return response["data"]["user"]
def check_streamer_online(self, streamer):
if time.time() < streamer.offline_at + 60:
@@ -175,7 +189,7 @@ class Twitch(object):
def __get_inventory(self):
response = self.post_gql_request(GQLOperations.Inventory)
return response["data"]["currentUser"]["inventory"]
return response["data"]["currentUser"]["inventory"] if response != {} else {}
def __get_drops_dashboard(self, status=None):
response = self.post_gql_request(GQLOperations.ViewerDropsDashboard)
@@ -292,14 +306,15 @@ class Twitch(object):
json_data["variables"] = {"channelLogin": streamer.username}
response = self.post_gql_request(json_data)
if response["data"]["community"] is None:
raise StreamerDoesNotExistException
channel = response["data"]["community"]["channel"]
community_points = channel["self"]["communityPoints"]
streamer.channel_points = community_points["balance"]
if response != {}:
if response["data"]["community"] is None:
raise StreamerDoesNotExistException
channel = response["data"]["community"]["channel"]
community_points = channel["self"]["communityPoints"]
streamer.channel_points = community_points["balance"]
if community_points["availableClaim"] is not None:
self.claim_bonus(streamer, community_points["availableClaim"]["id"])
if community_points["availableClaim"] is not None:
self.claim_bonus(streamer, community_points["availableClaim"]["id"])
def make_predictions(self, event):
decision = event.bet.calculate(event.streamer.channel_points)
@@ -320,21 +335,22 @@ class Twitch(object):
extra={"emoji": ":pushpin:"},
)
else:
logger.info(
f"Place {_millify(decision['amount'])} channel points on: {event.bet.get_outcome(selector_index)}",
extra={"emoji": ":four_leaf_clover:"},
)
if decision["amount"] > 0:
logger.info(
f"Place {_millify(decision['amount'])} channel points on: {event.bet.get_outcome(selector_index)}",
extra={"emoji": ":four_leaf_clover:"},
)
json_data = copy.deepcopy(GQLOperations.MakePrediction)
json_data["variables"] = {
"input": {
"eventID": event.event_id,
"outcomeID": decision["id"],
"points": decision["amount"],
"transactionID": token_hex(16),
json_data = copy.deepcopy(GQLOperations.MakePrediction)
json_data["variables"] = {
"input": {
"eventID": event.event_id,
"outcomeID": decision["id"],
"points": decision["amount"],
"transactionID": token_hex(16),
}
}
}
return self.post_gql_request(json_data)
return self.post_gql_request(json_data)
else:
logger.info(
f"Oh no! The event is not active anymore! Current status: {event.status}",
@@ -358,6 +374,12 @@ class Twitch(object):
)
]
for index in streamers_index:
if (streamers[index].stream.update_elapsed() / 60) > 10:
# Why this user It's currently online but the last updated was more than 10minutes ago?
# Please perform a manually update and check if the user it's online
self.check_streamer_online(streamers[index])
streamers_watching = []
for prior in priority:
if prior == Priority.ORDER and len(streamers_watching) < 2:
@@ -368,7 +390,6 @@ class Twitch(object):
Check if we need need to change priority based on watch streak
Viewers receive points for returning for x consecutive streams.
Each stream must be at least 10 minutes long and it must have been at least 30 minutes since the last stream ended.
Watch at least 6m for get the +10
"""
for index in streamers_index:
@@ -460,14 +481,23 @@ class Twitch(object):
)
except requests.exceptions.ConnectionError as e:
logger.error(f"Error while trying to watch a minute: {e}")
logger.error(f"Error while trying to send minute watched: {e}")
# The success rate It's very hight usually. Why we have failed?
# Check internet connection ...
while internet_connection_available() is False:
random_sleep = random.randint(1, 3)
logger.warning(
f"No internet connection available! Retry after {random_sleep}m"
)
time.sleep(random_sleep * 60)
self.__chuncked_sleep(
next_iteration - time.time(), chunk_size=chunk_size
)
if streamers_watching == []:
time.sleep(60)
self.__chuncked_sleep(60, chunk_size=chunk_size)
def get_channel_id(self, streamer_username):
json_response = self.__do_helix_request(f"/users?login={streamer_username}")
@@ -490,7 +520,7 @@ class Twitch(object):
json_response = self.__do_helix_request(query)
pagination = json_response["pagination"]
followers += [fw["to_name"].lower() for fw in json_response["data"]]
followers += [fw["to_login"].lower() for fw in json_response["data"]]
time.sleep(random.uniform(0.3, 0.7))
if pagination == {}:

11
TwitchChannelPointsMiner/classes/TwitchWebSocket.py
View File

@@ -2,7 +2,7 @@ import json
import logging
import time
from websocket import WebSocketApp
from websocket import WebSocketApp, WebSocketConnectionClosedException
from TwitchChannelPointsMiner.utils import create_nonce
@@ -52,9 +52,12 @@ class TwitchWebSocket(WebSocketApp):
self.last_ping = time.time()
def send(self, request):
request_str = json.dumps(request, separators=(",", ":"))
logger.debug(f"#{self.index} - Send: {request_str}")
super().send(request_str)
try:
request_str = json.dumps(request, separators=(",", ":"))
logger.debug(f"#{self.index} - Send: {request_str}")
super().send(request_str)
except WebSocketConnectionClosedException:
self.is_closed = True
def elapsed_last_pong(self):
return (time.time() - self.last_pong) // 60

109
TwitchChannelPointsMiner/classes/WebSocketsPool.py
View File

@@ -13,7 +13,11 @@ from TwitchChannelPointsMiner.classes.entities.Raid import Raid
# from TwitchChannelPointsMiner.classes.Exceptions import TimeBasedDropNotFound
from TwitchChannelPointsMiner.classes.TwitchWebSocket import TwitchWebSocket
from TwitchChannelPointsMiner.constants import WEBSOCKET
from TwitchChannelPointsMiner.utils import _millify, get_streamer_index
from TwitchChannelPointsMiner.utils import (
_millify,
get_streamer_index,
internet_connection_available,
)
logger = logging.getLogger(__name__)
@@ -33,58 +37,67 @@ class WebSocketsPool:
"""
def submit(self, topic):
if self.ws == [] or self.ws[-1] is None or len(self.ws[-1].topics) >= 50:
self.append_new_websocket()
# Check if we need to create a new WebSocket instance
if self.ws == [] or len(self.ws[-1].topics) >= 50:
self.ws.append(self.__new(len(self.ws)))
self.__start(-1)
self.ws[-1].topics.append(topic)
self.__submit(-1, topic)
if self.ws[-1].is_opened is False:
self.ws[-1].pending_topics.append(topic)
def __submit(self, index, topic):
# Topic in topics should never happen. Anyway prevent any types of duplicates
if topic not in self.ws[index].topics:
self.ws[index].topics.append(topic)
if self.ws[index].is_opened is False:
self.ws[index].pending_topics.append(topic)
else:
self.ws[-1].listen(topic, self.twitch.twitch_login.get_auth_token())
self.ws[index].listen(topic, self.twitch.twitch_login.get_auth_token())
def append_new_websocket(self):
self.ws.append(
TwitchWebSocket(
index=len(self.ws),
parent_pool=self,
url=WEBSOCKET,
on_message=WebSocketsPool.on_message,
on_open=WebSocketsPool.on_open,
on_error=WebSocketsPool.on_error,
on_close=WebSocketsPool.on_close
# on_close=WebSocketsPool.handle_reconnection, # Do nothing.
)
def __new(self, index):
return TwitchWebSocket(
index=index,
parent_pool=self,
url=WEBSOCKET,
on_message=WebSocketsPool.on_message,
on_open=WebSocketsPool.on_open,
on_error=WebSocketsPool.on_error,
on_close=WebSocketsPool.on_close
# on_close=WebSocketsPool.handle_reconnection, # Do nothing.
)
self.thread_ws = threading.Thread(target=lambda: self.ws[-1].run_forever())
self.thread_ws.daemon = True
self.thread_ws.start()
def __start(self, index):
thread_ws = threading.Thread(target=lambda: self.ws[index].run_forever())
thread_ws.daemon = True
thread_ws.name = f"WebSocket #{self.ws[index].index}"
thread_ws.start()
def end(self):
for index in range(0, len(self.ws)):
if self.ws[index] is not None:
self.ws[index].forced_close = True
self.ws[index].close()
self.ws[index].forced_close = True
self.ws[index].close()
@staticmethod
def on_open(ws):
def run():
ws.is_opened = True
ws.ping()
for topic in ws.pending_topics:
ws.listen(topic, ws.twitch.twitch_login.get_auth_token())
while not ws.is_closed:
ws.ping()
time.sleep(random.uniform(25, 30))
while ws.is_closed is False:
# Else: the ws is currently in reconnecting phase, you can't do ping or other operation.
# Probably this ws will be closed very soon with ws.is_closed = True
if ws.is_reconneting is False:
ws.ping() # We need ping for keep the connection alive
time.sleep(random.uniform(25, 30))
if ws.elapsed_last_pong() > 10 and ws.is_reconneting is False:
logger.info(
f"#{ws.index} - The last PONG was received more than 10 minutes ago. Reconnect the WebSocket"
)
ws.is_reconneting = True
WebSocketsPool.handle_reconnection(ws)
if ws.elapsed_last_pong() > 5:
logger.info(
f"#{ws.index} - The last PONG was received more than 5 minutes ago"
)
WebSocketsPool.handle_reconnection(ws)
thread_ws = threading.Thread(target=run)
thread_ws.daemon = True
@@ -92,6 +105,8 @@ class WebSocketsPool:
@staticmethod
def on_error(ws, error):
# Connection lost | [WinError 10054] An existing connection was forcibly closed by the remote host
# Connection already closed | Connection is already closed (raise WebSocketConnectionClosedException)
logger.error(f"#{ws.index} - WebSocket error: {error}")
@staticmethod
@@ -103,20 +118,37 @@ class WebSocketsPool:
@staticmethod
def handle_reconnection(ws):
# Close the current WebSocket.
# anyway, we replace the ws with None
ws.is_closed = True
ws.keep_running = False
# Reconnect only if ws.forced_close is False (replace the keep_running)
# Set the current socket as reconnecting status
# So the exsternal ping check will be locked
ws.is_reconneting = True
if ws.forced_close is False:
logger.info(
f"#{ws.index} - Reconnecting to Twitch PubSub server in 30 seconds"
f"#{ws.index} - Reconnecting to Twitch PubSub server in ~60 seconds"
)
time.sleep(30)
while internet_connection_available() is False:
random_sleep = random.randint(1, 3)
logger.warning(
f"#{ws.index} - No internet connection available! Retry after {random_sleep}m"
)
time.sleep(random_sleep * 60)
# Why not create a new ws on the same array index? Let's try.
self = ws.parent_pool
self.ws[ws.index] = None
self.ws[ws.index] = self.__new(ws.index) # Create a new connection.
# self.ws[ws.index].topics = ws.topics
self.__start(ws.index) # Start a new thread.
time.sleep(30)
for topic in ws.topics:
self.submit(topic)
self.__submit(ws.index, topic)
@staticmethod
def on_message(ws, message):
@@ -344,7 +376,6 @@ class WebSocketsPool:
elif response["type"] == "RECONNECT":
logger.info(f"#{ws.index} - Reconnection required")
ws.is_reconneting = True
WebSocketsPool.handle_reconnection(ws)
elif response["type"] == "PONG":

5
TwitchChannelPointsMiner/classes/entities/Stream.py
View File

@@ -66,7 +66,10 @@ class Stream(object):
return None if self.game in [{}, None] else self.game["name"]
def update_required(self):
return self.__last_update == 0 or (time.time() - self.__last_update) >= 120
return self.__last_update == 0 or self.update_elapsed() >= 120
def update_elapsed(self):
return 0 if self.__last_update == 0 else (time.time() - self.__last_update)
def init_watch_streak(self):
self.watch_streak_missing = True

10
TwitchChannelPointsMiner/utils.py
View File

@@ -1,5 +1,6 @@
import platform
import re
import socket
import time
from copy import deepcopy
from datetime import datetime, timezone
@@ -130,3 +131,12 @@ def set_default_settings(settings, defaults):
def char_decision_as_index(char):
return 0 if char == "A" else 1
def internet_connection_available(host="8.8.8.8", port=53, timeout=3):
try:
socket.setdefaulttimeout(timeout)
socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
return True
except socket.error:
return False