Add environment configuration and update main functionality
- Introduced .env.local for environment variables - Updated .gitignore to include new files - Enhanced main.py to utilize environment variables for user credentials - Implemented random sleep delay in API calls and output JSON formatting
This commit is contained in:
6
.env.local
Normal file
6
.env.local
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
OUTPUT_FILE=""
|
||||||
|
COOKIES=""
|
||||||
|
USERNAME=""
|
||||||
|
PASSWORD=""
|
||||||
|
EMAIL=""
|
||||||
|
EMAIL_PASSWORD=""
|
||||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,2 +1,4 @@
|
|||||||
.env
|
.env
|
||||||
.venv/
|
.venv/
|
||||||
|
accounts.db
|
||||||
|
tweets.json
|
||||||
43
main.py
43
main.py
@@ -1,48 +1,49 @@
|
|||||||
|
import asyncio
|
||||||
|
import random
|
||||||
|
from twscrape import API
|
||||||
import os
|
import os
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from twscrape import API, gather, set_log_level
|
|
||||||
|
|
||||||
|
|
||||||
# Configuration
|
# Configuration
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
OUTPUT_FILE = os.getenv("OUTPUT_FILE", "tweets.json")
|
OUTPUT_FILE = os.getenv("OUTPUT_FILE", "tweets.json")
|
||||||
SLEEP_MIN = 5
|
|
||||||
SLEEP_MAX = 10
|
|
||||||
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
|
|
||||||
|
|
||||||
logging.basicConfig(level=LOG_LEVEL, format="%(asctime)s %(levelname)s %(message)s")
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
COOKIES = {}
|
COOKIES = os.getenv("COOKIES")
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
|
|
||||||
api = API() # or API("path-to.db") – default is `accounts.db`
|
api = API() # or API("path-to.db") – default is `accounts.db`
|
||||||
|
|
||||||
# ADD ACCOUNTS (for CLI usage see next readme section)
|
# ADD ACCOUNTS (for CLI usage see next readme section)
|
||||||
|
|
||||||
# Option 1. Adding account with cookies (more stable)
|
# Option 1. Adding account with cookies (more stable)
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
cookies = os.getenv("COOKIES")
|
cookies = os.getenv("COOKIES")
|
||||||
await api.pool.add_account("user3", "pass3", "u3@mail.com", "mail_pass3", cookies=cookies)
|
username = os.getenv("USERNAME")
|
||||||
|
password = os.getenv("PASSWORD")
|
||||||
|
email = os.getenv("EMAIL")
|
||||||
|
email_password = os.getenv("EMAIL_PASSWORD")
|
||||||
|
|
||||||
|
await api.pool.add_account(username, password, email, email_password, cookies=cookies)
|
||||||
await api.pool.login_all() # try to login to receive account cookies
|
await api.pool.login_all() # try to login to receive account cookies
|
||||||
|
|
||||||
# NOTE 2: all methods have `raw` version (returns `httpx.Response` object):
|
# NOTE 2: all methods have `raw` version (returns `httpx.Response` object):
|
||||||
async for rep in api.search_raw("elon musk"):
|
async for rep in api.search("AI"):
|
||||||
print(rep.status_code, rep.json()) # rep is `httpx.Response` object
|
_results = []
|
||||||
|
try:
|
||||||
|
_results.append(rep.json())
|
||||||
|
except Exception:
|
||||||
|
_results.append(rep.text)
|
||||||
|
|
||||||
# change log level, default info
|
await asyncio.sleep(random.uniform(7, 15)) # random delay between 7 and 15 seconds
|
||||||
set_log_level("DEBUG")
|
|
||||||
|
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||||||
|
json.dump({"tweets": _results}, f, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
# Tweet & User model can be converted to regular dict or json, e.g.:
|
|
||||||
doc = await api.user_by_id(user_id) # User
|
|
||||||
doc.dict() # -> python dict
|
|
||||||
doc.json() # -> json string
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
Reference in New Issue
Block a user