base code added
This commit is contained in:
Utsho Dey
parent
1a1a713e0f
commit
d5b977e1d4
14 changed files with 1194 additions and 85 deletions
Binary file not shown.
|
|
@ -71,7 +71,6 @@ async def ai_fact_check(request: AIFactCheckRequest):
|
|||
|
||||
results[url] = verification_result
|
||||
all_sources.update(result["sources"])
|
||||
all_contexts.extend(result["context_used"])
|
||||
|
||||
# Accumulate token usage
|
||||
total_prompt_tokens += result["token_usage"]["prompt_tokens"]
|
||||
|
|
@ -88,7 +87,6 @@ async def ai_fact_check(request: AIFactCheckRequest):
|
|||
query=request.content,
|
||||
verification_result=results,
|
||||
sources=list(all_sources),
|
||||
context_used=all_contexts,
|
||||
token_usage=token_usage
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from app.models.fact_check_models import (
|
|||
TokenUsage
|
||||
)
|
||||
from app.websites.fact_checker_website import fetch_fact_checks, get_all_sources
|
||||
from app.api.scrap_websites import SearchRequest, search_websites
|
||||
|
||||
fact_check_router = APIRouter()
|
||||
|
||||
|
|
@ -33,7 +34,7 @@ class CustomJSONEncoder(json.JSONEncoder):
|
|||
)
|
||||
async def check_facts(request: FactCheckRequest) -> FactCheckResponse:
|
||||
"""
|
||||
Check facts using multiple fact-checking sources
|
||||
Check facts using multiple fact-checking sources and fallback to web search
|
||||
"""
|
||||
all_results = []
|
||||
verified_results = []
|
||||
|
|
@ -78,12 +79,55 @@ async def check_facts(request: FactCheckRequest) -> FactCheckResponse:
|
|||
contexts_used.append(review["textualRating"])
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
continue
|
||||
except Exception as e:
|
||||
# Log the error but continue with other sources
|
||||
print(f"Error processing {source.domain}: {str(e)}")
|
||||
continue
|
||||
|
||||
# If no results found, try searching websites
|
||||
if not all_results:
|
||||
try:
|
||||
# Create search request
|
||||
search_request = SearchRequest(
|
||||
search_text=request.content,
|
||||
source_types=["fact_checkers"]
|
||||
)
|
||||
|
||||
# Perform website search
|
||||
search_response = await search_websites(search_request)
|
||||
|
||||
# If AI fact check results are available, use them
|
||||
if search_response.ai_fact_check_result:
|
||||
# Create a claim from AI fact check result
|
||||
ai_claim = {
|
||||
"text": request.content,
|
||||
"claimant": "AI Analysis",
|
||||
"claimDate": datetime.now().isoformat(),
|
||||
"claimReview": [{
|
||||
"publisher": {
|
||||
"name": "AI Fact Checker",
|
||||
"site": "ai-fact-check"
|
||||
},
|
||||
"textualRating": search_response.ai_fact_check_result.verification_result["verdict"],
|
||||
"title": "AI Fact Check Analysis",
|
||||
"reviewDate": datetime.now().isoformat(),
|
||||
"url": ""
|
||||
}]
|
||||
}
|
||||
|
||||
validated_claim = Claim(**ai_claim).dict()
|
||||
all_results.append(validated_claim)
|
||||
|
||||
# Add sources and contexts
|
||||
all_sources_list.extend(search_response.results.keys())
|
||||
if search_response.ai_fact_check_result.verification_result["evidence"]:
|
||||
contexts_used.extend(search_response.ai_fact_check_result.verification_result["evidence"])
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during website search: {str(e)}")
|
||||
|
||||
# If still no results found after searching websites
|
||||
if not all_results:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
|
|
@ -99,7 +143,7 @@ async def check_facts(request: FactCheckRequest) -> FactCheckResponse:
|
|||
"verdict": "Insufficient Information", # Default verdict
|
||||
"confidence": "Low",
|
||||
"evidence": contexts_used,
|
||||
"reasoning": "Based on available fact checks",
|
||||
"reasoning": "Based on available fact checks and web search results",
|
||||
"missing_info": "Additional verification may be needed"
|
||||
}
|
||||
|
||||
|
|
@ -117,7 +161,6 @@ async def check_facts(request: FactCheckRequest) -> FactCheckResponse:
|
|||
results=all_results,
|
||||
verification_result=verification_result,
|
||||
sources=list(set(all_sources_list)),
|
||||
context_used=contexts_used,
|
||||
token_usage=token_usage,
|
||||
summary={
|
||||
"total_sources": len(set(all_sources_list)),
|
||||
|
|
|
|||
|
|
@ -1,18 +1,41 @@
|
|||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from typing import List, Dict, Optional
|
||||
from urllib.parse import urlencode, urlparse
|
||||
import urllib.parse
|
||||
import numpy as np
|
||||
from time import sleep
|
||||
import logging
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import re
|
||||
from app.services.openai_client import OpenAIClient
|
||||
from app.config import OPENAI_API_KEY
|
||||
from app.websites.fact_checker_website import SOURCES, get_all_sources
|
||||
from app.api.ai_fact_check import ai_fact_check
|
||||
from app.models.fact_check_models import (
|
||||
AIFactCheckRequest,
|
||||
AIFactCheckResponse,
|
||||
VerificationResult,
|
||||
TokenUsage
|
||||
)
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
scrap_websites_router = APIRouter()
|
||||
|
||||
# Configuration for scraping
|
||||
MAX_RETRIES = 2
|
||||
RETRY_DELAY = 2
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
search_text: str
|
||||
site_domains: List[str]
|
||||
source_types: List[str] = ["fact_checkers"]
|
||||
|
||||
class UrlSimilarityInfo(BaseModel):
|
||||
url: str
|
||||
|
|
@ -22,78 +45,121 @@ class UrlSimilarityInfo(BaseModel):
|
|||
class SearchResponse(BaseModel):
|
||||
results: Dict[str, List[str]]
|
||||
error_messages: Dict[str, str]
|
||||
url_similarities: Dict[str, List[UrlSimilarityInfo]]
|
||||
ai_fact_check_result: Optional[Dict] = None
|
||||
|
||||
def extract_url_text(url: str) -> str:
|
||||
"""Extract and process meaningful text from URL path with improved cleaning"""
|
||||
logger.debug(f"Extracting text from URL: {url}")
|
||||
try:
|
||||
# Parse the URL and get the path
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
path = parsed.path
|
||||
|
||||
# Remove common URL parts and file extensions
|
||||
path = path.replace('.html', '').replace('/index', '').replace('.php', '')
|
||||
|
||||
# Split path into segments
|
||||
segments = [seg for seg in path.split('/') if seg]
|
||||
|
||||
# Remove dates and numbers
|
||||
cleaned_segments = []
|
||||
for segment in segments:
|
||||
# Replace hyphens and underscores with spaces
|
||||
segment = segment.replace('-', ' ').replace('_', ' ')
|
||||
|
||||
# Filter out segments that are just dates or numbers
|
||||
if not (segment.replace(' ', '').isdigit() or
|
||||
all(part.isdigit() for part in segment.split() if part)):
|
||||
cleaned_segments.append(segment)
|
||||
|
||||
# Remove very common words that don't add meaning
|
||||
common_words = {
|
||||
'www', 'live', 'news', 'intl', 'index', 'world', 'us', 'uk',
|
||||
'updates', 'update', 'latest', 'breaking', 'new', 'article'
|
||||
}
|
||||
|
||||
# Join segments and split into words
|
||||
text = ' '.join(cleaned_segments)
|
||||
words = [word.lower() for word in text.split()
|
||||
if word.lower() not in common_words and len(word) > 1]
|
||||
|
||||
return ' '.join(words)
|
||||
except Exception:
|
||||
result = ' '.join(words)
|
||||
logger.debug(f"Extracted text: {result}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting text from URL {url}: {str(e)}")
|
||||
return ''
|
||||
|
||||
def google_search_scraper(search_text: str, site_domain: str) -> List[str]:
|
||||
query = f"{search_text} \"site:{site_domain}\""
|
||||
encoded_query = urllib.parse.quote(query)
|
||||
base_url = "https://www.google.com/search"
|
||||
url = f"{base_url}?q={encoded_query}"
|
||||
def extract_search_results(html_content):
|
||||
"""Extract URLs using multiple selectors and patterns"""
|
||||
soup = BeautifulSoup(html_content, 'html.parser')
|
||||
urls = set() # Using set to avoid duplicates
|
||||
|
||||
# Multiple CSS selectors to try
|
||||
selectors = [
|
||||
'div.g div.yuRUbf > a', # Main result links
|
||||
'div.g a.l', # Alternative link format
|
||||
'div.rc a', # Another possible format
|
||||
'div[class*="g"] > div > div > div > a', # Broader match
|
||||
'a[href^="http"]' # Any http link
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
try:
|
||||
elements = soup.select(selector)
|
||||
for element in elements:
|
||||
url = element.get('href')
|
||||
if url and url.startswith('http') and not url.startswith('https://www.google.com'):
|
||||
urls.add(url)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error with selector {selector}: {str(e)}")
|
||||
|
||||
# Also try finding URLs in the raw HTML using regex
|
||||
url_pattern = r'href="(https?://[^"]+)"'
|
||||
raw_urls = re.findall(url_pattern, html_content)
|
||||
for url in raw_urls:
|
||||
if not url.startswith('https://www.google.com'):
|
||||
urls.add(url)
|
||||
|
||||
return list(urls)
|
||||
|
||||
def google_search_scraper(search_text: str, site_domain: str, retry_count: int = 0) -> List[str]:
|
||||
"""Scrape Google search results with multiple query formats"""
|
||||
logger.info(f"Searching for '{search_text}' on domain: {site_domain}")
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Referer': 'https://www.google.com/',
|
||||
'DNT': '1'
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
search_results = soup.find_all('div', class_='g')
|
||||
|
||||
urls = []
|
||||
for result in search_results[:5]:
|
||||
link = result.find('a')
|
||||
if link and 'href' in link.attrs:
|
||||
url = link['href']
|
||||
if url.startswith('http'):
|
||||
urls.append(url)
|
||||
|
||||
return urls[:5]
|
||||
|
||||
except requests.RequestException as e:
|
||||
raise HTTPException(status_code=500, detail=f"Error scraping {site_domain}: {str(e)}")
|
||||
# Try different query formats
|
||||
query_formats = [
|
||||
f"{search_text} site:{site_domain}",
|
||||
f"site:{site_domain} {search_text}",
|
||||
f"\"{search_text}\" site:{site_domain}"
|
||||
]
|
||||
|
||||
all_urls = set()
|
||||
|
||||
for query in query_formats:
|
||||
try:
|
||||
google_url = f"https://www.google.com/search?q={urlencode({'q': query})}"
|
||||
logger.debug(f"Trying query format: {query}")
|
||||
|
||||
response = requests.get(google_url, headers=headers)
|
||||
|
||||
if response.status_code == 200:
|
||||
urls = extract_search_results(response.text)
|
||||
domain_urls = [url for url in urls if site_domain in urlparse(url).netloc]
|
||||
all_urls.update(domain_urls)
|
||||
else:
|
||||
logger.warning(f"Received status code {response.status_code} for query format: {query}")
|
||||
|
||||
sleep(2) # Delay between requests
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing query format '{query}': {str(e)}")
|
||||
if retry_count < MAX_RETRIES:
|
||||
sleep(RETRY_DELAY)
|
||||
return google_search_scraper(search_text, site_domain, retry_count + 1)
|
||||
|
||||
valid_urls = list(all_urls)
|
||||
logger.info(f"Found {len(valid_urls)} unique URLs for domain: {site_domain}")
|
||||
return valid_urls[:5] # Return up to 5 URLs
|
||||
|
||||
def calculate_similarity(query_embedding: List[float], url_embedding: List[float]) -> float:
|
||||
"""Calculate cosine similarity between two embeddings"""
|
||||
query_array = np.array(query_embedding)
|
||||
url_array = np.array(url_embedding)
|
||||
|
||||
|
|
@ -102,59 +168,142 @@ def calculate_similarity(query_embedding: List[float], url_embedding: List[float
|
|||
)
|
||||
return float(similarity)
|
||||
|
||||
|
||||
@scrap_websites_router.post("/search", response_model=SearchResponse)
|
||||
async def search_websites(request: SearchRequest):
|
||||
logger.info(f"Starting search with query: {request.search_text}")
|
||||
logger.info(f"Source types requested: {request.source_types}")
|
||||
|
||||
results = {}
|
||||
error_messages = {}
|
||||
url_similarities = {}
|
||||
|
||||
# Initialize OpenAI client
|
||||
logger.debug("Initializing OpenAI client")
|
||||
openai_client = OpenAIClient(OPENAI_API_KEY)
|
||||
|
||||
# Get domains based on requested source types
|
||||
domains = []
|
||||
for source_type in request.source_types:
|
||||
if source_type in SOURCES:
|
||||
domains.extend([source.domain for source in SOURCES[source_type]])
|
||||
|
||||
if not domains:
|
||||
logger.warning("No valid source types provided. Using all available domains.")
|
||||
domains = [source.domain for source in get_all_sources()]
|
||||
|
||||
logger.info(f"Processing {len(domains)} domains")
|
||||
|
||||
# Enhance search text with key terms
|
||||
search_context = request.search_text
|
||||
logger.debug("Getting query embedding from OpenAI")
|
||||
query_embedding = openai_client.get_embeddings([search_context])[0]
|
||||
|
||||
# Higher similarity threshold for better filtering
|
||||
SIMILARITY_THRESHOLD = 0.75
|
||||
MAX_URLS_PER_DOMAIN = 2 # Adjusted to ensure total stays under 5
|
||||
TOTAL_MAX_URLS = 5 # Maximum URLs allowed for AIFactCheckRequest
|
||||
|
||||
for domain in request.site_domains:
|
||||
total_urls_collected = 0
|
||||
for domain in domains[:3]: # Limit to 3 domains for testing
|
||||
if total_urls_collected >= TOTAL_MAX_URLS:
|
||||
break
|
||||
|
||||
logger.info(f"Processing domain: {domain}")
|
||||
try:
|
||||
urls = google_search_scraper(request.search_text, domain)
|
||||
url_sims = []
|
||||
valid_urls = []
|
||||
|
||||
logger.debug(f"Found {len(urls)} URLs for domain {domain}")
|
||||
|
||||
for url in urls:
|
||||
if len(valid_urls) >= MAX_URLS_PER_DOMAIN or total_urls_collected >= TOTAL_MAX_URLS:
|
||||
break
|
||||
|
||||
url_text = extract_url_text(url)
|
||||
|
||||
# Skip URLs with no meaningful text extracted
|
||||
if not url_text:
|
||||
logger.debug(f"No meaningful text extracted from URL: {url}")
|
||||
continue
|
||||
|
||||
|
||||
logger.debug("Getting URL embedding from OpenAI")
|
||||
url_embedding = openai_client.get_embeddings([url_text])[0]
|
||||
similarity = calculate_similarity(query_embedding, url_embedding)
|
||||
|
||||
url_sims.append(UrlSimilarityInfo(
|
||||
url=url,
|
||||
similarity=similarity,
|
||||
extracted_text=url_text
|
||||
))
|
||||
logger.debug(f"Similarity score for {url}: {similarity}")
|
||||
|
||||
if similarity >= SIMILARITY_THRESHOLD:
|
||||
valid_urls.append(url)
|
||||
total_urls_collected += 1
|
||||
|
||||
results[domain] = valid_urls
|
||||
url_similarities[domain] = sorted(url_sims,
|
||||
key=lambda x: x.similarity,
|
||||
reverse=True)
|
||||
logger.info(f"Successfully processed domain {domain}. Found {len(valid_urls)} valid URLs")
|
||||
|
||||
except HTTPException as e:
|
||||
logger.error(f"HTTP Exception for domain {domain}: {str(e.detail)}")
|
||||
error_messages[domain] = str(e.detail)
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error for domain {domain}: {str(e)}")
|
||||
error_messages[domain] = f"Unexpected error for {domain}: {str(e)}"
|
||||
|
||||
sleep(1) # Add delay between processing different domains
|
||||
|
||||
logger.info("Search completed")
|
||||
logger.debug(f"Results found for {len(results)} domains")
|
||||
logger.debug(f"Errors encountered for {len(error_messages)} domains")
|
||||
|
||||
# Collect all valid URLs from results
|
||||
all_valid_urls = []
|
||||
for domain_urls in results.values():
|
||||
all_valid_urls.extend(domain_urls)
|
||||
|
||||
logger.info(f"Total valid URLs collected: {len(all_valid_urls)}")
|
||||
|
||||
# Create request body for AI fact check
|
||||
if all_valid_urls:
|
||||
fact_check_request = AIFactCheckRequest(
|
||||
content=request.search_text,
|
||||
urls=all_valid_urls[:TOTAL_MAX_URLS] # Ensure we don't exceed the limit
|
||||
)
|
||||
|
||||
logger.info("Calling AI fact check service")
|
||||
try:
|
||||
ai_response = await ai_fact_check(fact_check_request)
|
||||
logger.info("AI fact check completed successfully")
|
||||
|
||||
# Format AI fact check response
|
||||
formatted_response = {
|
||||
"query": ai_response.query,
|
||||
"token_usage": {
|
||||
"prompt_tokens": ai_response.token_usage.prompt_tokens,
|
||||
"completion_tokens": ai_response.token_usage.completion_tokens,
|
||||
"total_tokens": ai_response.token_usage.total_tokens
|
||||
},
|
||||
"sources": ai_response.sources,
|
||||
"verification_result": {
|
||||
url: {
|
||||
"verdict": result.verdict,
|
||||
"confidence": result.confidence,
|
||||
"evidence": result.evidence,
|
||||
"reasoning": result.reasoning,
|
||||
"missing_info": result.missing_info
|
||||
} for url, result in ai_response.verification_result.items()
|
||||
}
|
||||
}
|
||||
|
||||
# Return response with AI fact check results
|
||||
return SearchResponse(
|
||||
results=results,
|
||||
error_messages=error_messages,
|
||||
ai_fact_check_result=formatted_response
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during AI fact check: {str(e)}")
|
||||
error_messages["ai_fact_check"] = f"Error during fact checking: {str(e)}"
|
||||
|
||||
# Return response without AI fact check if no valid URLs or error occurred
|
||||
return SearchResponse(
|
||||
results=results,
|
||||
error_messages=error_messages,
|
||||
url_similarities=url_similarities
|
||||
ai_fact_check_result=None
|
||||
)
|
||||
261
app/api/scrap_websites2.py
Normal file
261
app/api/scrap_websites2.py
Normal file
|
|
@ -0,0 +1,261 @@
|
|||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Optional
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import urllib.parse
|
||||
import numpy as np
|
||||
from time import sleep
|
||||
import logging
|
||||
from app.services.openai_client import OpenAIClient
|
||||
from app.config import OPENAI_API_KEY
|
||||
from app.websites.fact_checker_website import SOURCES, get_all_sources
|
||||
from app.api.ai_fact_check import ai_fact_check
|
||||
from app.models.fact_check_models import AIFactCheckRequest, AIFactCheckResponse
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
scrap_websites_router = APIRouter()
|
||||
|
||||
# Configuration for rate limiting
|
||||
RATE_LIMIT_DELAY = 2 # Delay between requests in seconds
|
||||
MAX_RETRIES = 1 # Maximum number of retries per domain
|
||||
RETRY_DELAY = 1 # Delay between retries in seconds
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
search_text: str
|
||||
source_types: List[str] = ["fact_checkers"]
|
||||
|
||||
class UrlSimilarityInfo(BaseModel):
|
||||
url: str
|
||||
similarity: float
|
||||
extracted_text: str
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
results: Dict[str, List[str]]
|
||||
error_messages: Dict[str, str]
|
||||
ai_fact_check_result: Optional[AIFactCheckResponse] = None
|
||||
|
||||
def extract_url_text(url: str) -> str:
|
||||
"""Extract and process meaningful text from URL path with improved cleaning"""
|
||||
logger.debug(f"Extracting text from URL: {url}")
|
||||
try:
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
path = parsed.path
|
||||
path = path.replace('.html', '').replace('/index', '').replace('.php', '')
|
||||
segments = [seg for seg in path.split('/') if seg]
|
||||
cleaned_segments = []
|
||||
for segment in segments:
|
||||
segment = segment.replace('-', ' ').replace('_', ' ')
|
||||
if not (segment.replace(' ', '').isdigit() or
|
||||
all(part.isdigit() for part in segment.split() if part)):
|
||||
cleaned_segments.append(segment)
|
||||
|
||||
common_words = {
|
||||
'www', 'live', 'news', 'intl', 'index', 'world', 'us', 'uk',
|
||||
'updates', 'update', 'latest', 'breaking', 'new', 'article'
|
||||
}
|
||||
|
||||
text = ' '.join(cleaned_segments)
|
||||
words = [word.lower() for word in text.split()
|
||||
if word.lower() not in common_words and len(word) > 1]
|
||||
|
||||
result = ' '.join(words)
|
||||
logger.debug(f"Extracted text: {result}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting text from URL {url}: {str(e)}")
|
||||
return ''
|
||||
|
||||
def google_search_scraper(search_text: str, site_domain: str, retry_count: int = 0) -> List[str]:
|
||||
"""Scrape Google search results with retry logic and rate limiting"""
|
||||
logger.info(f"Searching for '{search_text}' on domain: {site_domain} (Attempt {retry_count + 1}/{MAX_RETRIES})")
|
||||
|
||||
if retry_count >= MAX_RETRIES:
|
||||
logger.error(f"Max retries exceeded for domain: {site_domain}")
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"Max retries exceeded for {site_domain}"
|
||||
)
|
||||
|
||||
query = f"{search_text} \"site:{site_domain}\""
|
||||
encoded_query = urllib.parse.quote(query)
|
||||
base_url = "https://www.google.com/search"
|
||||
url = f"{base_url}?q={encoded_query}"
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
}
|
||||
|
||||
try:
|
||||
logger.debug(f"Waiting {RATE_LIMIT_DELAY} seconds before request")
|
||||
sleep(RATE_LIMIT_DELAY)
|
||||
|
||||
logger.debug(f"Making request to Google Search for domain: {site_domain}")
|
||||
response = requests.get(url, headers=headers)
|
||||
|
||||
if response.status_code == 429 or "sorry/index" in response.url:
|
||||
logger.warning(f"Rate limit hit for domain {site_domain}. Retrying after delay...")
|
||||
sleep(RETRY_DELAY)
|
||||
return google_search_scraper(search_text, site_domain, retry_count + 1)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
search_results = soup.find_all('div', class_='g')
|
||||
|
||||
urls = []
|
||||
for result in search_results[:3]:
|
||||
link = result.find('a')
|
||||
if link and 'href' in link.attrs:
|
||||
url = link['href']
|
||||
if url.startswith('http'):
|
||||
urls.append(url)
|
||||
|
||||
logger.info(f"Found {len(urls)} results for domain: {site_domain}")
|
||||
return urls[:5]
|
||||
|
||||
except requests.RequestException as e:
|
||||
if retry_count < MAX_RETRIES:
|
||||
logger.warning(f"Request failed for {site_domain}. Retrying... Error: {str(e)}")
|
||||
sleep(RETRY_DELAY)
|
||||
return google_search_scraper(search_text, site_domain, retry_count + 1)
|
||||
logger.error(f"All retries failed for domain {site_domain}. Error: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Error scraping {site_domain}: {str(e)}"
|
||||
)
|
||||
|
||||
def calculate_similarity(query_embedding: List[float], url_embedding: List[float]) -> float:
|
||||
"""Calculate cosine similarity between two embeddings"""
|
||||
query_array = np.array(query_embedding)
|
||||
url_array = np.array(url_embedding)
|
||||
|
||||
similarity = np.dot(url_array, query_array) / (
|
||||
np.linalg.norm(url_array) * np.linalg.norm(query_array)
|
||||
)
|
||||
return float(similarity)
|
||||
|
||||
@scrap_websites_router.post("/search", response_model=SearchResponse)
|
||||
async def search_websites(request: SearchRequest):
|
||||
logger.info(f"Starting search with query: {request.search_text}")
|
||||
logger.info(f"Source types requested: {request.source_types}")
|
||||
|
||||
results = {}
|
||||
error_messages = {}
|
||||
url_similarities = {}
|
||||
|
||||
# Initialize OpenAI client
|
||||
logger.debug("Initializing OpenAI client")
|
||||
openai_client = OpenAIClient(OPENAI_API_KEY)
|
||||
|
||||
# Get domains based on requested source types
|
||||
domains = []
|
||||
for source_type in request.source_types:
|
||||
if source_type in SOURCES:
|
||||
domains.extend([source.domain for source in SOURCES[source_type]])
|
||||
|
||||
if not domains:
|
||||
logger.warning("No valid source types provided. Using all available domains.")
|
||||
domains = [source.domain for source in get_all_sources()]
|
||||
|
||||
logger.info(f"Processing {len(domains)} domains")
|
||||
|
||||
# Enhance search text with key terms
|
||||
search_context = request.search_text
|
||||
logger.debug("Getting query embedding from OpenAI")
|
||||
query_embedding = openai_client.get_embeddings([search_context])[0]
|
||||
|
||||
# Higher similarity threshold for better filtering
|
||||
SIMILARITY_THRESHOLD = 0.75
|
||||
|
||||
for domain in domains:
|
||||
logger.info(f"Processing domain: {domain}")
|
||||
try:
|
||||
urls = google_search_scraper(request.search_text, domain)
|
||||
url_sims = []
|
||||
valid_urls = []
|
||||
|
||||
logger.debug(f"Found {len(urls)} URLs for domain {domain}")
|
||||
|
||||
for url in urls:
|
||||
url_text = extract_url_text(url)
|
||||
|
||||
if not url_text:
|
||||
logger.debug(f"No meaningful text extracted from URL: {url}")
|
||||
continue
|
||||
|
||||
logger.debug("Getting URL embedding from OpenAI")
|
||||
url_embedding = openai_client.get_embeddings([url_text])[0]
|
||||
similarity = calculate_similarity(query_embedding, url_embedding)
|
||||
|
||||
logger.debug(f"Similarity score for {url}: {similarity}")
|
||||
|
||||
url_sims.append(UrlSimilarityInfo(
|
||||
url=url,
|
||||
similarity=similarity,
|
||||
extracted_text=url_text
|
||||
))
|
||||
|
||||
if similarity >= SIMILARITY_THRESHOLD:
|
||||
valid_urls.append(url)
|
||||
|
||||
results[domain] = valid_urls
|
||||
url_similarities[domain] = sorted(url_sims,
|
||||
key=lambda x: x.similarity,
|
||||
reverse=True)
|
||||
|
||||
logger.info(f"Successfully processed domain {domain}. Found {len(valid_urls)} valid URLs")
|
||||
|
||||
except HTTPException as e:
|
||||
logger.error(f"HTTP Exception for domain {domain}: {str(e.detail)}")
|
||||
error_messages[domain] = str(e.detail)
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error for domain {domain}: {str(e)}")
|
||||
error_messages[domain] = f"Unexpected error for {domain}: {str(e)}"
|
||||
|
||||
logger.info("Search completed")
|
||||
logger.debug(f"Results found for {len(results)} domains")
|
||||
logger.debug(f"Errors encountered for {len(error_messages)} domains")
|
||||
|
||||
# Collect all valid URLs from results
|
||||
all_valid_urls = []
|
||||
for domain_urls in results.values():
|
||||
all_valid_urls.extend(domain_urls)
|
||||
|
||||
logger.info(f"Total valid URLs collected: {len(all_valid_urls)}")
|
||||
|
||||
# Create request body for AI fact check
|
||||
if all_valid_urls:
|
||||
fact_check_request = AIFactCheckRequest(
|
||||
content=request.search_text,
|
||||
urls=all_valid_urls
|
||||
)
|
||||
|
||||
logger.info("Calling AI fact check service")
|
||||
try:
|
||||
ai_response = await ai_fact_check(fact_check_request)
|
||||
logger.info("AI fact check completed successfully")
|
||||
|
||||
# Return response with AI fact check results
|
||||
return SearchResponse(
|
||||
results=results,
|
||||
error_messages=error_messages,
|
||||
ai_fact_check_result=ai_response
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during AI fact check: {str(e)}")
|
||||
error_messages["ai_fact_check"] = f"Error during fact checking: {str(e)}"
|
||||
|
||||
# Return response without AI fact check if no valid URLs or error occurred
|
||||
return SearchResponse(
|
||||
results=results,
|
||||
error_messages=error_messages,
|
||||
ai_fact_check_result=None
|
||||
)
|
||||
Binary file not shown.
|
|
@ -140,7 +140,6 @@ class BaseFactCheckResponse(BaseModel):
|
|||
query: str
|
||||
token_usage: TokenUsage
|
||||
sources: List[str]
|
||||
context_used: List[str]
|
||||
|
||||
model_config = ConfigDict(json_schema_extra={
|
||||
"example": {
|
||||
|
|
@ -151,7 +150,6 @@ class BaseFactCheckResponse(BaseModel):
|
|||
"total_tokens": 150
|
||||
},
|
||||
"sources": ["source1.com", "source2.com"],
|
||||
"context_used": ["Relevant context from sources"]
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -183,7 +181,6 @@ class GoogleFactCheckResponse(BaseFactCheckResponse):
|
|||
"reasoning": "Detailed analysis"
|
||||
},
|
||||
"sources": ["factchecker.com"],
|
||||
"context_used": ["Relevant context"],
|
||||
"token_usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
|
|
@ -219,10 +216,6 @@ class AIFactCheckResponse(BaseFactCheckResponse):
|
|||
}
|
||||
},
|
||||
"sources": ["source1.com", "source2.com"],
|
||||
"context_used": [
|
||||
"Context from source 1",
|
||||
"Context from source 2"
|
||||
],
|
||||
"token_usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 100,
|
||||
|
|
|
|||
|
|
@ -142,7 +142,6 @@ class AIFactChecker:
|
|||
return {
|
||||
"verification_result": response["response"], # This is now a dictionary
|
||||
"sources": sources,
|
||||
"context_used": [doc.page_content for doc in relevant_docs],
|
||||
"token_usage": {
|
||||
"prompt_tokens": response["prompt_tokens"],
|
||||
"completion_tokens": response["completion_tokens"],
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -8,21 +8,8 @@ SOURCES = {
|
|||
"fact_checkers": [
|
||||
FactCheckSource(domain=domain, type=SourceType.FACT_CHECKER, priority=1)
|
||||
for domain in [
|
||||
"factcheck.org",
|
||||
"snopes.com",
|
||||
"politifact.com",
|
||||
"reuters.com",
|
||||
"bbc.com",
|
||||
"apnews.com",
|
||||
"usatoday.com",
|
||||
"nytimes.com",
|
||||
"washingtonpost.com",
|
||||
"afp.com",
|
||||
"fullfact.org",
|
||||
"truthorfiction.com",
|
||||
"leadstories.com",
|
||||
"altnews.in",
|
||||
"boomlive.in",
|
||||
"en.prothomalo.com"
|
||||
]
|
||||
],
|
||||
|
|
|
|||
595
output.json
Normal file
595
output.json
Normal file
|
|
@ -0,0 +1,595 @@
|
|||
{
|
||||
"kind": "customsearch#search",
|
||||
"url": {
|
||||
"type": "application/json",
|
||||
"template": "https://www.googleapis.com/customsearch/v1?q={searchTerms}&num={count?}&start={startIndex?}&lr={language?}&safe={safe?}&cx={cx?}&sort={sort?}&filter={filter?}&gl={gl?}&cr={cr?}&googlehost={googleHost?}&c2coff={disableCnTwTranslation?}&hq={hq?}&hl={hl?}&siteSearch={siteSearch?}&siteSearchFilter={siteSearchFilter?}&exactTerms={exactTerms?}&excludeTerms={excludeTerms?}&linkSite={linkSite?}&orTerms={orTerms?}&dateRestrict={dateRestrict?}&lowRange={lowRange?}&highRange={highRange?}&searchType={searchType}&fileType={fileType?}&rights={rights?}&imgSize={imgSize?}&imgType={imgType?}&imgColorType={imgColorType?}&imgDominantColor={imgDominantColor?}&alt=json"
|
||||
},
|
||||
"queries": {
|
||||
"request": [
|
||||
{
|
||||
"title": "Google Custom Search - Sheikh Hasina resigned as a Prime Minister of Bangladesh",
|
||||
"totalResults": "758000",
|
||||
"searchTerms": "Sheikh Hasina resigned as a Prime Minister of Bangladesh",
|
||||
"count": 10,
|
||||
"startIndex": 1,
|
||||
"inputEncoding": "utf8",
|
||||
"outputEncoding": "utf8",
|
||||
"safe": "off",
|
||||
"cx": "d437f1eb581de4590"
|
||||
}
|
||||
],
|
||||
"nextPage": [
|
||||
{
|
||||
"title": "Google Custom Search - Sheikh Hasina resigned as a Prime Minister of Bangladesh",
|
||||
"totalResults": "758000",
|
||||
"searchTerms": "Sheikh Hasina resigned as a Prime Minister of Bangladesh",
|
||||
"count": 10,
|
||||
"startIndex": 11,
|
||||
"inputEncoding": "utf8",
|
||||
"outputEncoding": "utf8",
|
||||
"safe": "off",
|
||||
"cx": "d437f1eb581de4590"
|
||||
}
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"title": "Prothom Alo"
|
||||
},
|
||||
"searchInformation": {
|
||||
"searchTime": 0.513164,
|
||||
"formattedSearchTime": "0.51",
|
||||
"totalResults": "758000",
|
||||
"formattedTotalResults": "758,000"
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Sheikh Hasina: Euphoria in Bangladesh after PM flees country",
|
||||
"htmlTitle": "\u003cb\u003eSheikh Hasina\u003c/b\u003e: Euphoria in \u003cb\u003eBangladesh\u003c/b\u003e after PM flees country",
|
||||
"link": "https://www.bbc.com/news/articles/clywww69p2vo",
|
||||
"displayLink": "www.bbc.com",
|
||||
"snippet": "Aug 5, 2024 ... Bangladeshi Prime Minister Sheikh Hasina has resigned after weeks of deadly anti-government protests, putting an end to her more than two decades dominating ...",
|
||||
"htmlSnippet": "Aug 5, 2024 \u003cb\u003e...\u003c/b\u003e \u003cb\u003eBangladeshi Prime Minister Sheikh Hasina\u003c/b\u003e has \u003cb\u003eresigned\u003c/b\u003e after weeks of deadly anti-government protests, putting an end to her more than two decades dominating ...",
|
||||
"formattedUrl": "https://www.bbc.com/news/articles/clywww69p2vo",
|
||||
"htmlFormattedUrl": "https://www.bbc.com/news/articles/clywww69p2vo",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ2noEFH2T-yJo4oB7DU_MF2FqAUzIHU5paMXHka1ny_vMi037f2gtOZ3of&s",
|
||||
"width": "300",
|
||||
"height": "168"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"msapplication-tilecolor": "#da532c",
|
||||
"og:image": "https://ichef.bbci.co.uk/news/1024/branded_news/db85/live/388ebc30-5367-11ef-aebc-6de4d31bf5cd.jpg",
|
||||
"apple-itunes-app": "app-id=364147881, app-argument=https://www.bbc.com/news/articles/clywww69p2vo",
|
||||
"twitter:title": "Sheikh Hasina: Euphoria in Bangladesh after PM flees country",
|
||||
"twitter:card": "summary_large_image",
|
||||
"og:image:alt": "Protesters storming Prime Minister Sheikh Hasina's palace after she fled the country",
|
||||
"theme-color": "#ffffff",
|
||||
"al:ios:app_name": "BBC: World News & Stories",
|
||||
"og:title": "Sheikh Hasina: Euphoria in Bangladesh after PM flees country",
|
||||
"al:android:package": "bbc.mobile.news.ww",
|
||||
"al:ios:url": "bbcx://news/articles/clywww69p2vo",
|
||||
"al:web:url": "https://bbc.com/news/articles/clywww69p2vo",
|
||||
"og:description": "President Mohammed Shahabuddin ordered the release of a jailed former prime minister.",
|
||||
"version": "2.12.0+20",
|
||||
"al:ios:app_store_id": "364147881",
|
||||
"twitter:image:src": "https://ichef.bbci.co.uk/news/1024/branded_news/db85/live/388ebc30-5367-11ef-aebc-6de4d31bf5cd.jpg",
|
||||
"al:android:url": "bbcx://news/articles/clywww69p2vo",
|
||||
"next-head-count": "36",
|
||||
"twitter:image:alt": "Protesters storming Prime Minister Sheikh Hasina's palace after she fled the country",
|
||||
"viewport": "width=device-width",
|
||||
"twitter:description": "President Mohammed Shahabuddin ordered the release of a jailed former prime minister.",
|
||||
"al:android:app_name": "BBC: World News & Stories"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://ichef.bbci.co.uk/news/1024/branded_news/db85/live/388ebc30-5367-11ef-aebc-6de4d31bf5cd.jpg"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Bangladesh: Prime Minister Hasina Resigns amid Mass Protests ...",
|
||||
"htmlTitle": "\u003cb\u003eBangladesh\u003c/b\u003e: \u003cb\u003ePrime Minister Hasina Resigns\u003c/b\u003e amid Mass Protests ...",
|
||||
"link": "https://www.hrw.org/news/2024/08/06/bangladesh-prime-minister-hasina-resigns-amid-mass-protests",
|
||||
"displayLink": "www.hrw.org",
|
||||
"snippet": "Aug 6, 2024 ... (London) – Bangladesh Prime Minister Sheikh Hasina resigned on August 5, 2024, and fled the country after weeks of student protests, ...",
|
||||
"htmlSnippet": "Aug 6, 2024 \u003cb\u003e...\u003c/b\u003e (London) – \u003cb\u003eBangladesh Prime Minister Sheikh Hasina resigned\u003c/b\u003e on August 5, 2024, and fled the country after weeks of student protests, ...",
|
||||
"formattedUrl": "https://www.hrw.org/.../bangladesh-prime-minister-hasina-resigns-amid-ma...",
|
||||
"htmlFormattedUrl": "https://www.hrw.org/.../\u003cb\u003ebangladesh\u003c/b\u003e-\u003cb\u003eprime\u003c/b\u003e-\u003cb\u003eminister\u003c/b\u003e-\u003cb\u003ehasina\u003c/b\u003e-resigns-amid-ma...",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcT7Rd-kZwml7ax4Q_93QFbon2bmbwYliEYvMil6qgM0xEG6tV72lS_iclM&s",
|
||||
"width": "310",
|
||||
"height": "163"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"og:image": "https://www.hrw.org/sites/default/files/styles/opengraph/public/media_2024/08/202408asia_bangladesh_Sheikh%20Hasina.jpg?h=888143e8&itok=IKUTUc3F",
|
||||
"og:image:alt": "Bangladesh’s former Prime Minister Sheikh Hasina addresses the media in Mirpur after the anti-quota protests.",
|
||||
"article:published_time": "2024-08-06T14:00:00-0400",
|
||||
"twitter:card": "summary_large_image",
|
||||
"twitter:title": "Bangladesh: Prime Minister Hasina Resigns amid Mass Protests",
|
||||
"og:site_name": "Human Rights Watch",
|
||||
"twitter:site:id": "14700316",
|
||||
"handheldfriendly": "true",
|
||||
"og:title": "Bangladesh: Prime Minister Hasina Resigns amid Mass Protests",
|
||||
"google": "H_DzcJuJMJKVAO6atlPsK4HHr2WienspT6e74P5fVFY",
|
||||
"og:updated_time": "2024-08-08T10:24:02-0400",
|
||||
"og:description": "Bangladesh Prime Minister Sheikh Hasina resigned on August 5, 2024, and fled the country after weeks of student protests.",
|
||||
"og:image:secure_url": "https://www.hrw.org/sites/default/files/styles/opengraph/public/media_2024/08/202408asia_bangladesh_Sheikh%20Hasina.jpg?h=888143e8&itok=IKUTUc3F",
|
||||
"article:publisher": "https://www.facebook.com/HumanRightsWatch",
|
||||
"twitter:image": "https://www.hrw.org/sites/default/files/styles/opengraph/public/media_2024/08/202408asia_bangladesh_Sheikh%20Hasina.jpg?h=888143e8&itok=IKUTUc3F",
|
||||
"twitter:image:alt": "Bangladesh’s former Prime Minister Sheikh Hasina addresses the media in Mirpur after the anti-quota protests.",
|
||||
"twitter:site": "@hrw",
|
||||
"article:modified_time": "2024-08-08T10:24:02-0400",
|
||||
"viewport": "width=device-width, initial-scale=1.0",
|
||||
"twitter:description": "Bangladesh Prime Minister Sheikh Hasina resigned on August 5, 2024, and fled the country after weeks of student protests.",
|
||||
"mobileoptimized": "width",
|
||||
"og:url": "https://www.hrw.org/news/2024/08/06/bangladesh-prime-minister-hasina-resigns-amid-mass-protests"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://www.hrw.org/sites/default/files/styles/opengraph/public/media_2024/08/202408asia_bangladesh_Sheikh%20Hasina.jpg?h=888143e8&itok=IKUTUc3F"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Bangladesh wakes up to new uncertain future after PM Sheikh ...",
|
||||
"htmlTitle": "\u003cb\u003eBangladesh\u003c/b\u003e wakes up to new uncertain future after PM \u003cb\u003eSheikh\u003c/b\u003e ...",
|
||||
"link": "https://www.bbc.com/news/live/ckdgg87lnkdt",
|
||||
"displayLink": "www.bbc.com",
|
||||
"snippet": "Aug 5, 2024 ... Yesterday's historic events saw Bangladesh's Prime Minister Sheikh Hasina resign from power and flee the country. Today, government ...",
|
||||
"htmlSnippet": "Aug 5, 2024 \u003cb\u003e...\u003c/b\u003e Yesterday's historic events saw \u003cb\u003eBangladesh's Prime Minister Sheikh Hasina resign\u003c/b\u003e from power and flee the country. Today, government ...",
|
||||
"formattedUrl": "https://www.bbc.com/news/live/ckdgg87lnkdt",
|
||||
"htmlFormattedUrl": "https://www.bbc.com/news/live/ckdgg87lnkdt",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ9V5V2pFKUOVvlosPa5swslIzMQnDiFW21RkSxNXvXxhrcyvRNZMc2bqXE&s",
|
||||
"width": "300",
|
||||
"height": "168"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"og:image": "https://static.files.bbci.co.uk/ws/simorgh-assets/public/news/images/metadata/poster-1024x576.png",
|
||||
"theme-color": "#FFFFFF",
|
||||
"og:type": "article",
|
||||
"twitter:title": "Bangladesh wakes up to new uncertain future after PM Sheikh Hasina's dramatic resignation",
|
||||
"og:site_name": "BBC News",
|
||||
"twitter:url": "https://www.bbc.com/news/live/ckdgg87lnkdt",
|
||||
"og:title": "Bangladesh wakes up to new uncertain future after PM Sheikh Hasina's dramatic resignation",
|
||||
"msapplication-tileimage": "https://static.files.bbci.co.uk/core/website/assets/static/icons/windows-phone/news/windows-phone-icon-270x270.23502b4459eb7a6ab2ab.png",
|
||||
"og:description": "Looting and disorder have been reported in the South Asian nation, a day after mass protests forced Ms Hasina to flee and resign.",
|
||||
"fb:pages": "1143803202301544,317278538359186,1392506827668140,742734325867560,185246968166196,156060587793370,137920769558355,193435954068976,21263239760,156400551056385,929399697073756,154344434967,228735667216,80758950658,260212261199,294662213128,1086451581439054,283348121682053,295830058648,239931389545417,304314573046,310719525611571,647687225371774,1159932557403143,286567251709437,1731770190373618,125309456546,163571453661989,285361880228,512423982152360,238003846549831,176663550714,260967092113,118450564909230,100978706649892,15286229625,122103087870579,120655094632228,102814153147070,124715648647,153132638110668,150467675018739",
|
||||
"twitter:creator": "@BBCWorld",
|
||||
"article:author": "https://www.facebook.com/bbcnews",
|
||||
"twitter:image": "https://static.files.bbci.co.uk/ws/simorgh-assets/public/news/images/metadata/poster-1024x576.png",
|
||||
"fb:app_id": "1609039196070050",
|
||||
"twitter:site": "@BBCWorld",
|
||||
"viewport": "width=device-width, initial-scale=1",
|
||||
"twitter:description": "Looting and disorder have been reported in the South Asian nation, a day after mass protests forced Ms Hasina to flee and resign.",
|
||||
"og:locale": "en_GB",
|
||||
"og:image_alt": "BBC News",
|
||||
"fb:admins": "100004154058350",
|
||||
"og:url": "https://www.bbc.com/news/live/ckdgg87lnkdt",
|
||||
"format-detection": "telephone=no"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://static.files.bbci.co.uk/ws/simorgh-assets/public/news/images/metadata/poster-1024x576.png"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Bangladesh protests: PM Sheikh Hasina flees to India as ...",
|
||||
"htmlTitle": "\u003cb\u003eBangladesh\u003c/b\u003e protests: PM \u003cb\u003eSheikh Hasina\u003c/b\u003e flees to India as ...",
|
||||
"link": "https://www.cnn.com/2024/08/05/asia/bangladesh-prime-minister-residence-stormed-intl/index.html",
|
||||
"displayLink": "www.cnn.com",
|
||||
"snippet": "Aug 6, 2024 ... The prime minister of Bangladesh, Sheikh Hasina, resigned and fled to neighboring India on Monday after protesters stormed her official ...",
|
||||
"htmlSnippet": "Aug 6, 2024 \u003cb\u003e...\u003c/b\u003e The \u003cb\u003eprime minister of Bangladesh\u003c/b\u003e, \u003cb\u003eSheikh Hasina\u003c/b\u003e, \u003cb\u003eresigned\u003c/b\u003e and fled to neighboring India on Monday after protesters stormed her official ...",
|
||||
"formattedUrl": "https://www.cnn.com/2024/08/05/.../bangladesh-prime-minister.../index.ht...",
|
||||
"htmlFormattedUrl": "https://www.cnn.com/2024/08/05/.../\u003cb\u003ebangladesh\u003c/b\u003e-\u003cb\u003eprime\u003c/b\u003e-\u003cb\u003eminister\u003c/b\u003e.../index.ht...",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcScyayfP1an0tjs821kLSqSGIsgUFwc02vkRXh6ERXuqeV7xOEt3sC__sM&s",
|
||||
"width": "300",
|
||||
"height": "168"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"og:image": "https://media.cnn.com/api/v1/images/stellar/prod/ap24218390125876-2.jpg?c=16x9&q=w_800,c_fill",
|
||||
"twitter:title": "Bangladesh prime minister flees to India as anti-government protesters storm her residence | CNN",
|
||||
"og:type": "article",
|
||||
"twitter:card": "summary_large_image",
|
||||
"article:published_time": "2024-08-05T10:01:00.074Z",
|
||||
"og:site_name": "CNN",
|
||||
"author": "Isaac Yee, Tanbirul Miraj Ripon",
|
||||
"og:title": "Bangladesh prime minister flees to India as anti-government protesters storm her residence | CNN",
|
||||
"meta-section": "world",
|
||||
"type": "article",
|
||||
"og:description": "The prime minister of Bangladesh, Sheikh Hasina, resigned and fled to neighboring India on Monday after protesters stormed her official residence after weeks of deadly anti-government demonstrations in the South Asian nation.",
|
||||
"twitter:image": "https://media.cnn.com/api/v1/images/stellar/prod/ap24218390125876-2.jpg?c=16x9&q=w_800,c_fill",
|
||||
"article:publisher": "https://www.facebook.com/CNN",
|
||||
"fb:app_id": "80401312489",
|
||||
"twitter:site": "@CNN",
|
||||
"article:modified_time": "2024-08-06T05:24:05.249Z",
|
||||
"viewport": "width=device-width,initial-scale=1,shrink-to-fit=no",
|
||||
"twitter:description": "The prime minister of Bangladesh, Sheikh Hasina, resigned and fled to neighboring India on Monday after protesters stormed her official residence after weeks of deadly anti-government demonstrations in the South Asian nation.",
|
||||
"template_type": "article_leaf",
|
||||
"theme": "world",
|
||||
"og:url": "https://www.cnn.com/2024/08/05/asia/bangladesh-prime-minister-residence-stormed-intl/index.html"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://media.cnn.com/api/v1/images/stellar/prod/ap24218390125876-2.jpg?c=16x9&q=w_800,c_fill"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Why did Bangladesh PM Sheikh Hasina resign and where is she ...",
|
||||
"htmlTitle": "Why did \u003cb\u003eBangladesh\u003c/b\u003e PM \u003cb\u003eSheikh Hasina resign\u003c/b\u003e and where is she ...",
|
||||
"link": "https://www.reuters.com/world/asia-pacific/why-did-bangladesh-pm-sheikh-hasina-resign-where-is-she-now-2024-08-06/",
|
||||
"displayLink": "www.reuters.com",
|
||||
"snippet": "Aug 6, 2024 ... Aug 7 (Reuters) - Sheikh Hasina resigned as Bangladesh's prime minister and fled the country on Monday following weeks of dedly protests ...",
|
||||
"htmlSnippet": "Aug 6, 2024 \u003cb\u003e...\u003c/b\u003e Aug 7 (Reuters) - \u003cb\u003eSheikh Hasina resigned\u003c/b\u003e as \u003cb\u003eBangladesh's prime minister\u003c/b\u003e and fled the country on Monday following weeks of dedly protests ...",
|
||||
"formattedUrl": "https://www.reuters.com/.../why-did-bangladesh-pm-sheikh-hasina-resign-...",
|
||||
"htmlFormattedUrl": "https://www.reuters.com/.../why-did-\u003cb\u003ebangladesh\u003c/b\u003e-pm-\u003cb\u003esheikh\u003c/b\u003e-\u003cb\u003ehasina\u003c/b\u003e-resign-...",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcR_IDuyjGdce77t1tWrSwheC6g8XSyuUQKn_KxA0H9x3eCRV4kretMyY0_J&s",
|
||||
"width": "310",
|
||||
"height": "162"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"apple-itunes-app": "app-id=602660809, app-argument=https://www.reuters.com/world/asia-pacific/why-did-bangladesh-pm-sheikh-hasina-resign-where-is-she-now-2024-08-06/?id=E5O5XBJMZBPTDAUM7I6BFYX4UA",
|
||||
"og:image": "https://www.reuters.com/resizer/v2/QSLRZINWOVJ25LWOYIOOXO4L6A.jpg?auth=581b869970d6c61101b4e8bba552bd5ae55ec08c8a333a33ef63a72f57b8f0c4&height=1005&width=1920&quality=80&smart=true",
|
||||
"analytics:page_layout": "regular-article",
|
||||
"article:published_time": "2024-08-07T03:23:35Z",
|
||||
"og:image:width": "1200",
|
||||
"twitter:card": "summary_large_image",
|
||||
"og:site_name": "Reuters",
|
||||
"og:article:modified_time": "2024-08-07T03:51:39.907Z",
|
||||
"ccbot": "nofollow",
|
||||
"analytics:ad_layout": "leaderboard, right rail, sponsored",
|
||||
"analyticsattributes.topicchannel": "World",
|
||||
"title": "Why did Bangladesh PM Sheikh Hasina resign and where is she now? | Reuters",
|
||||
"og:description": "Sheikh Hasina resigned as Bangladesh's prime minister and fled the country on Monday following weeks of dedly protests that began as demonstrations by students against government job quotas but surged into a movement demanding her resignation.",
|
||||
"twitter:creator": "@Reuters",
|
||||
"twitter:image": "https://www.reuters.com/resizer/v2/QSLRZINWOVJ25LWOYIOOXO4L6A.jpg?auth=581b869970d6c61101b4e8bba552bd5ae55ec08c8a333a33ef63a72f57b8f0c4&height=1005&width=1920&quality=80&smart=true",
|
||||
"twitter:image:alt": "Bangladeshi Prime Minister Sheikh Hasina reviews an honour guard at the Government House, during her visit to Thailand, in Bangkok, Thailand, April 26, 2024. REUTERS/Athit Perawongmetha/File Photo",
|
||||
"twitter:site": "@Reuters",
|
||||
"article:modified_time": "2024-08-07T03:51:39.907Z",
|
||||
"fb:admins": "988502044532272",
|
||||
"article:content_tier": "metered",
|
||||
"og:type": "article",
|
||||
"article:section": "Asia Pacific",
|
||||
"og:image:alt": "Bangladeshi Prime Minister Sheikh Hasina reviews an honour guard at the Government House, during her visit to Thailand, in Bangkok, Thailand, April 26, 2024. REUTERS/Athit Perawongmetha/File Photo",
|
||||
"twitter:title": "Why did Bangladesh PM Sheikh Hasina resign and where is she now?",
|
||||
"ad:template": "article",
|
||||
"og:image:url": "https://www.reuters.com/resizer/v2/QSLRZINWOVJ25LWOYIOOXO4L6A.jpg?auth=581b869970d6c61101b4e8bba552bd5ae55ec08c8a333a33ef63a72f57b8f0c4&height=1005&width=1920&quality=80&smart=true",
|
||||
"dcsext.dartzone": "/4735792/reuters.com/world/apac/article",
|
||||
"og:title": "Why did Bangladesh PM Sheikh Hasina resign and where is she now?",
|
||||
"dcsext.channellist": "World;World;Asia Pacific;Asian Markets",
|
||||
"og:image:height": "628",
|
||||
"og:article:published_time": "2024-08-07T03:23:35Z",
|
||||
"og:updated_time": "2024-08-07T03:51:39.907Z",
|
||||
"fb:pages": "114050161948682",
|
||||
"article:author": "Sudipto Ganguly",
|
||||
"article:tag": "MTVID,EXPLN,TOPNWS,ANLINS,CIV,CWP,DIP,DLI,ECI,ECO,EDU,GEN,JOB,MCE,MPLT,MPOP,NEWS1,POL,RACR,SOCI,TOPCMB,VIO,SASIA,IN,PK,ASXPAC,BD,EMRG,ASIA,PACKAGE:US-TOP-NEWS,PACKAGE:WORLD-NEWS",
|
||||
"analyticsattributes.topicsubchannel": "Asia Pacific",
|
||||
"fb:app_id": "988502044532272",
|
||||
"og:locale:alternate": "en_US",
|
||||
"viewport": "width=device-width, initial-scale=1",
|
||||
"twitter:description": "Sheikh Hasina resigned as Bangladesh's prime minister and fled the country on Monday following weeks of dedly protests that began as demonstrations by students against government job quotas but surged into a movement demanding her resignation.",
|
||||
"og:locale": "en_US",
|
||||
"og:url": "https://www.reuters.com/world/asia-pacific/why-did-bangladesh-pm-sheikh-hasina-resign-where-is-she-now-2024-08-06/"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://www.reuters.com/resizer/v2/QSLRZINWOVJ25LWOYIOOXO4L6A.jpg?auth=581b869970d6c61101b4e8bba552bd5ae55ec08c8a333a33ef63a72f57b8f0c4&height=1005&width=1920&quality=80&smart=true"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Bangladesh's 'Gen Z revolution' toppled PM Sheikh Hasina. Why did ...",
|
||||
"htmlTitle": "\u003cb\u003eBangladesh's\u003c/b\u003e 'Gen Z revolution' toppled PM \u003cb\u003eSheikh Hasina\u003c/b\u003e. Why did ...",
|
||||
"link": "https://www.cnn.com/2024/08/06/asia/bangladesh-protests-hasina-resignation-explainer-intl-hnk/index.html",
|
||||
"displayLink": "www.cnn.com",
|
||||
"snippet": "Aug 6, 2024 ... People celebrate the resignation of Prime Minister Sheikh Hasina in Dhaka, Bangladesh, on August 5, 2024. Mohammad Ponir Hossain/Reuters. CNN —.",
|
||||
"htmlSnippet": "Aug 6, 2024 \u003cb\u003e...\u003c/b\u003e People celebrate the \u003cb\u003eresignation\u003c/b\u003e of \u003cb\u003ePrime Minister Sheikh Hasina\u003c/b\u003e in Dhaka, \u003cb\u003eBangladesh\u003c/b\u003e, on August 5, 2024. Mohammad Ponir Hossain/Reuters. CNN —.",
|
||||
"formattedUrl": "https://www.cnn.com/2024/08/06/asia/bangladesh...hasina.../index.html",
|
||||
"htmlFormattedUrl": "https://www.cnn.com/2024/08/06/asia/\u003cb\u003ebangladesh\u003c/b\u003e...\u003cb\u003ehasina\u003c/b\u003e.../index.html",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTOW5T__EO6GShxs6es-aGavTBFUU2GCU-SyqlBE3t5d0hFX5WugbjKA-JH&s",
|
||||
"width": "300",
|
||||
"height": "168"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"og:image": "https://media.cnn.com/api/v1/images/stellar/prod/2024-08-05t184829z-2105365796-rc2l99a18sqr-rtrmadp-3-bangladesh-protests.jpg?c=16x9&q=w_800,c_fill",
|
||||
"twitter:title": "Bangladesh’s ‘Gen Z revolution’ toppled a veteran leader. Why did they hit the streets and what happens now? | CNN",
|
||||
"og:type": "article",
|
||||
"twitter:card": "summary_large_image",
|
||||
"article:published_time": "2024-08-06T08:16:31.519Z",
|
||||
"og:site_name": "CNN",
|
||||
"author": "Helen Regan",
|
||||
"og:title": "Bangladesh’s ‘Gen Z revolution’ toppled a veteran leader. Why did they hit the streets and what happens now? | CNN",
|
||||
"meta-section": "world",
|
||||
"type": "article",
|
||||
"og:description": "Inside Bangladesh it’s being dubbed a Gen Z revolution – a protest movement that pitted mostly young student demonstrators against a 76-year-old leader who had dominated her nation for decades and turned increasingly authoritarian in recent years.",
|
||||
"twitter:image": "https://media.cnn.com/api/v1/images/stellar/prod/2024-08-05t184829z-2105365796-rc2l99a18sqr-rtrmadp-3-bangladesh-protests.jpg?c=16x9&q=w_800,c_fill",
|
||||
"article:publisher": "https://www.facebook.com/CNN",
|
||||
"fb:app_id": "80401312489",
|
||||
"twitter:site": "@CNN",
|
||||
"article:modified_time": "2024-08-07T03:48:11.066Z",
|
||||
"viewport": "width=device-width,initial-scale=1,shrink-to-fit=no",
|
||||
"twitter:description": "Inside Bangladesh it’s being dubbed a Gen Z revolution – a protest movement that pitted mostly young student demonstrators against a 76-year-old leader who had dominated her nation for decades and turned increasingly authoritarian in recent years.",
|
||||
"template_type": "article_leaf",
|
||||
"theme": "world",
|
||||
"og:url": "https://www.cnn.com/2024/08/06/asia/bangladesh-protests-hasina-resignation-explainer-intl-hnk/index.html"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://media.cnn.com/api/v1/images/stellar/prod/2024-08-05t184829z-2105365796-rc2l99a18sqr-rtrmadp-3-bangladesh-protests.jpg?c=16x9&q=w_800,c_fill"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Bangladesh PM Sheikh Hasina resigns, ending 15 years in power ...",
|
||||
"htmlTitle": "\u003cb\u003eBangladesh\u003c/b\u003e PM \u003cb\u003eSheikh Hasina resigns\u003c/b\u003e, ending 15 years in power ...",
|
||||
"link": "https://www.npr.org/2024/08/05/g-s1-15332/bangladesh-protests",
|
||||
"displayLink": "www.npr.org",
|
||||
"snippet": "Aug 5, 2024 ... DHAKA, Bangladesh — Bangladesh's Prime Minister Sheikh Hasina resigned on Monday, ending 15 years in power as thousands of protesters defied ...",
|
||||
"htmlSnippet": "Aug 5, 2024 \u003cb\u003e...\u003c/b\u003e DHAKA, \u003cb\u003eBangladesh\u003c/b\u003e — \u003cb\u003eBangladesh's Prime Minister Sheikh Hasina resigned\u003c/b\u003e on Monday, ending 15 years in power as thousands of protesters defied ...",
|
||||
"formattedUrl": "https://www.npr.org/2024/08/05/g-s1-15332/bangladesh-protests",
|
||||
"htmlFormattedUrl": "https://www.npr.org/2024/08/05/g-s1-15332/\u003cb\u003ebangladesh\u003c/b\u003e-protests",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSqvTrTl13trd-nrF4oQvAQOY3z2N2MfxSSyZsmd4Pm6E_e0TTbu0ER6zE&s",
|
||||
"width": "300",
|
||||
"height": "168"
|
||||
}
|
||||
],
|
||||
"speakablespecification": [
|
||||
{
|
||||
"cssselector": "[data-is-speakable]"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"date": "2024-08-05",
|
||||
"apple-itunes-app": "app-id=324906251, app-argument=https://www.npr.org/2024/08/05/g-s1-15332/bangladesh-protests",
|
||||
"og:image": "https://npr.brightspotcdn.com/dims3/default/strip/false/crop/6043x3399+0+315/resize/1400/quality/100/format/jpeg/?url=http%3A%2F%2Fnpr-brightspot.s3.amazonaws.com%2Fba%2F99%2Ff772f9bd44ee9b1ddf5a4d9d1d98%2Fap24217447347066.jpg",
|
||||
"og:type": "article",
|
||||
"twitter:card": "summary_large_image",
|
||||
"twitter:title": "Bangladesh PM Sheikh Hasina resigns, ending 15 years in power, as thousands protest",
|
||||
"og:site_name": "NPR",
|
||||
"cxenseparse:pageclass": "article",
|
||||
"twitter:domain": "npr.org",
|
||||
"cxenseparse:publishtime": "2024-08-05T04:07:23-04:00",
|
||||
"og:title": "Bangladesh PM Sheikh Hasina resigns, ending 15 years in power, as thousands protest",
|
||||
"rating": "General",
|
||||
"og:description": "At least 95 people, including at least 14 police officers, died in clashes in the capital on Sunday. Broadband internet and mobile data services were cut off for about three hours on Monday.",
|
||||
"fb:pages": "10643211755",
|
||||
"twitter:image:src": "https://npr.brightspotcdn.com/dims3/default/strip/false/crop/6043x3399+0+315/resize/1400/quality/100/format/jpeg/?url=http%3A%2F%2Fnpr-brightspot.s3.amazonaws.com%2Fba%2F99%2Ff772f9bd44ee9b1ddf5a4d9d1d98%2Fap24217447347066.jpg",
|
||||
"fb:app_id": "138837436154588",
|
||||
"cxenseparse:author": "The Associated Press",
|
||||
"twitter:site": "@NPR",
|
||||
"article:modified_time": "2024-08-05T06:50:55-04:00",
|
||||
"viewport": "width=device-width, initial-scale=1, shrink-to-fit=no",
|
||||
"article:content_tier": "free",
|
||||
"og:url": "https://www.npr.org/2024/08/05/g-s1-15332/bangladesh-protests",
|
||||
"article:opinion": "false"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://npr.brightspotcdn.com/dims3/default/strip/false/crop/6043x3399+0+315/resize/1400/quality/100/format/jpeg/?url=http%3A%2F%2Fnpr-brightspot.s3.amazonaws.com%2Fba%2F99%2Ff772f9bd44ee9b1ddf5a4d9d1d98%2Fap24217447347066.jpg"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Tens of thousands protest in Bangladesh to demand resignation of ...",
|
||||
"htmlTitle": "Tens of thousands protest in \u003cb\u003eBangladesh\u003c/b\u003e to demand \u003cb\u003eresignation\u003c/b\u003e of ...",
|
||||
"link": "https://www.cnn.com/2022/12/11/asia/bangladesh-protests-prime-minister-sheikh-hasina-intl-hnk/index.html",
|
||||
"displayLink": "www.cnn.com",
|
||||
"snippet": "Dec 11, 2022 ... Supporters of Bangladesh's opposition party protest against the government of Prime Minister Sheikh Hasina on December 10, 2022. Mamunur Rashid/ ...",
|
||||
"htmlSnippet": "Dec 11, 2022 \u003cb\u003e...\u003c/b\u003e Supporters of \u003cb\u003eBangladesh's\u003c/b\u003e opposition party protest against the government of \u003cb\u003ePrime Minister Sheikh Hasina\u003c/b\u003e on December 10, 2022. Mamunur Rashid/ ...",
|
||||
"formattedUrl": "https://www.cnn.com/.../bangladesh...prime-minister-sheikh-hasina.../index....",
|
||||
"htmlFormattedUrl": "https://www.cnn.com/.../\u003cb\u003ebangladesh\u003c/b\u003e...\u003cb\u003eprime\u003c/b\u003e-\u003cb\u003eminister\u003c/b\u003e-\u003cb\u003esheikh\u003c/b\u003e-\u003cb\u003ehasina\u003c/b\u003e.../index....",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ-JqzYxoZHcQ5wWQhH5Xq-JrKFFyWbdfS339bDlIrhMrc2Y_9BznDwjN5u&s",
|
||||
"width": "275",
|
||||
"height": "183"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"og:image": "https://media.cnn.com/api/v1/images/stellar/prod/221210230748-02-dhaka-protests-121022.jpg?c=16x9&q=w_800,c_fill",
|
||||
"twitter:title": "Tens of thousands protest in Bangladesh to demand resignation of Prime Minister | CNN",
|
||||
"og:type": "article",
|
||||
"twitter:card": "summary_large_image",
|
||||
"article:published_time": "2022-12-11T06:09:58Z",
|
||||
"og:site_name": "CNN",
|
||||
"author": "Vedika Sud,Yong Xiong",
|
||||
"og:title": "Tens of thousands protest in Bangladesh to demand resignation of Prime Minister | CNN",
|
||||
"meta-section": "world",
|
||||
"type": "article",
|
||||
"og:description": "Tens of thousands of protesters took to the streets of Dhaka on Saturday calling for the dissolution of parliament to make way for new elections, and demand the resignation of Bangladeshi Prime Minister Sheikh Hasina.",
|
||||
"twitter:image": "https://media.cnn.com/api/v1/images/stellar/prod/221210230748-02-dhaka-protests-121022.jpg?c=16x9&q=w_800,c_fill",
|
||||
"article:publisher": "https://www.facebook.com/CNN",
|
||||
"article:tag": "asia, bangladesh, brand safety-nsf other, brand safety-nsf sensitive, british national party, civil disobedience, continents and regions, domestic alerts, domestic-international news, elections and campaigns, government and public administration, iab-elections, iab-politics, political figures - intl, political organizations, political parties - intl, politics, protests and demonstrations, resignations, sheikh hasina, society, south asia",
|
||||
"fb:app_id": "80401312489",
|
||||
"twitter:site": "@CNN",
|
||||
"article:modified_time": "2022-12-11T06:09:58Z",
|
||||
"viewport": "width=device-width,initial-scale=1,shrink-to-fit=no",
|
||||
"twitter:description": "Tens of thousands of protesters took to the streets of Dhaka on Saturday calling for the dissolution of parliament to make way for new elections, and demand the resignation of Bangladeshi Prime Minister Sheikh Hasina.",
|
||||
"template_type": "article_leaf",
|
||||
"theme": "world",
|
||||
"og:url": "https://www.cnn.com/2022/12/11/asia/bangladesh-protests-prime-minister-sheikh-hasina-intl-hnk/index.html"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://media.cnn.com/api/v1/images/stellar/prod/221210230749-dhaka-protests-221207.jpg?q=w_1110,c_fill"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Timeline of events leading to the resignation of Bangladesh Prime ...",
|
||||
"htmlTitle": "Timeline of events leading to the \u003cb\u003eresignation\u003c/b\u003e of \u003cb\u003eBangladesh Prime\u003c/b\u003e ...",
|
||||
"link": "https://www.voanews.com/a/timeline-of-events-leading-to-the-resignation-of-bangladesh-prime-minister-sheikh-hasina/7731456.html",
|
||||
"displayLink": "www.voanews.com",
|
||||
"snippet": "Aug 5, 2024 ... Bangladesh Prime Minister Sheikh Hasina resigned and left the country Monday after clashes between student protesters and police left nearly 300 people dead.",
|
||||
"htmlSnippet": "Aug 5, 2024 \u003cb\u003e...\u003c/b\u003e \u003cb\u003eBangladesh Prime Minister Sheikh Hasina resigned\u003c/b\u003e and left the \u003cb\u003ecountry Monday\u003c/b\u003e after clashes between student protesters and police left nearly 300 people dead.",
|
||||
"formattedUrl": "https://www.voanews.com/...bangladesh-prime-minister-sheikh-hasina/7731...",
|
||||
"htmlFormattedUrl": "https://www.voanews.com/...\u003cb\u003ebangladesh\u003c/b\u003e-\u003cb\u003eprime\u003c/b\u003e-\u003cb\u003eminister\u003c/b\u003e-\u003cb\u003esheikh\u003c/b\u003e-\u003cb\u003ehasina\u003c/b\u003e/7731...",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS2o9D0XbnmDtsmWEVzDYCwGv4IHKkzATikOvXEDghsD_uzZj-G6_63zGyR&s",
|
||||
"width": "311",
|
||||
"height": "162"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"msapplication-tilecolor": "#ffffff",
|
||||
"apple-itunes-app": "app-id=632618796, app-argument=//7731456.ltr",
|
||||
"og:image": "https://gdb.voanews.com/28CFE9FB-9B7B-4474-8342-C7BC5434B54A.jpg",
|
||||
"og:type": "article",
|
||||
"og:image:width": "308",
|
||||
"twitter:card": "summary_large_image",
|
||||
"og:site_name": "Voice of America",
|
||||
"msvalidate.01": "3286EE554B6F672A6F2E608C02343C0E",
|
||||
"author": "Sabir Mustafa",
|
||||
"apple-mobile-web-app-title": "VOA",
|
||||
"og:title": "Timeline of events leading to the resignation of Bangladesh Prime Minister Sheikh Hasina",
|
||||
"msapplication-tileimage": "/Content/responsive/VOA/img/webApp/ico-144x144.png",
|
||||
"fb:pages": "36235438073",
|
||||
"og:description": "Hasina resigns after weeks of clashes between student protesters and police leave nearly 300 dead",
|
||||
"article:publisher": "https://www.facebook.com/voiceofamerica",
|
||||
"twitter:image": "https://gdb.voanews.com/28CFE9FB-9B7B-4474-8342-C7BC5434B54A.jpg",
|
||||
"fb:app_id": "362002700549372",
|
||||
"apple-mobile-web-app-status-bar-style": "black",
|
||||
"twitter:site": "@voanews",
|
||||
"viewport": "width=device-width, initial-scale=1.0",
|
||||
"twitter:description": "Hasina resigns after weeks of clashes between student protesters and police leave nearly 300 dead",
|
||||
"og:url": "https://www.voanews.com/a/timeline-of-events-leading-to-the-resignation-of-bangladesh-prime-minister-sheikh-hasina/7731456.html"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://gdb.voanews.com/28CFE9FB-9B7B-4474-8342-C7BC5434B54A.jpg"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "customsearch#result",
|
||||
"title": "Bangladesh's Sheikh Hasina forced to resign: What happened and ...",
|
||||
"htmlTitle": "\u003cb\u003eBangladesh's Sheikh Hasina\u003c/b\u003e forced to \u003cb\u003eresign\u003c/b\u003e: What happened and ...",
|
||||
"link": "https://www.aljazeera.com/news/2024/8/5/bangladeshs-sheikh-hasina-forced-to-resign-what-happened-and-whats-next",
|
||||
"displayLink": "www.aljazeera.com",
|
||||
"snippet": "Aug 5, 2024 ... Bangladesh Prime Minister Sheikh Hasina has stepped down from office, ending 15 years of what the opposition says was “authoritarian rule” and sparking ...",
|
||||
"htmlSnippet": "Aug 5, 2024 \u003cb\u003e...\u003c/b\u003e \u003cb\u003eBangladesh Prime Minister Sheikh Hasina\u003c/b\u003e has \u003cb\u003estepped down\u003c/b\u003e from office, ending 15 years of what the opposition says was “authoritarian rule” and sparking ...",
|
||||
"formattedUrl": "https://www.aljazeera.com/.../bangladeshs-sheikh-hasina-forced-to-resign-w...",
|
||||
"htmlFormattedUrl": "https://www.aljazeera.com/.../\u003cb\u003ebangladesh\u003c/b\u003es-\u003cb\u003esheikh\u003c/b\u003e-\u003cb\u003ehasina\u003c/b\u003e-forced-to-resign-w...",
|
||||
"pagemap": {
|
||||
"cse_thumbnail": [
|
||||
{
|
||||
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS2uyLUKVFCDpJ-_MjZ6dRKW5_LC1zknAIICxM5ZcVuAZtYqupigTOI_l_0&s",
|
||||
"width": "259",
|
||||
"height": "194"
|
||||
}
|
||||
],
|
||||
"metatags": [
|
||||
{
|
||||
"pagetype": "Article Page",
|
||||
"og:image": "https://www.aljazeera.com/wp-content/uploads/2024/08/AP24218390076912-1722855595.jpg?resize=1920%2C1440",
|
||||
"apple-itunes-app": "app-id=1534955972",
|
||||
"twitter:card": "summary_large_image",
|
||||
"og:site_name": "Al Jazeera",
|
||||
"postlabel": "Explainer",
|
||||
"twitter:url": "https://www.aljazeera.com/news/2024/8/5/bangladeshs-sheikh-hasina-forced-to-resign-what-happened-and-whats-next",
|
||||
"pagesection": "Explainer,News,Sheikh Hasina",
|
||||
"channel": "aje",
|
||||
"publisheddate": "2024-08-05T15:14:49",
|
||||
"postid": "3096869",
|
||||
"source": "Al Jazeera",
|
||||
"og:description": "Prime minister reportedly flees to India after weeks of antigovernment protests.",
|
||||
"taxonomyterms": "News, Sheikh Hasina, Asia, Bangladesh",
|
||||
"lastdate": "2024-08-05T15:40:26",
|
||||
"primarytopic": "News",
|
||||
"twitter:image:alt": "Sheikh Hasina forced to resign: What happened and what’s next?",
|
||||
"sourcetaxonomy": "Al Jazeera",
|
||||
"internalreporting": "Break it down for me",
|
||||
"where": "Asia, Bangladesh",
|
||||
"primarytag": "Sheikh Hasina",
|
||||
"ga4": "G-XN9JB9Q0M1",
|
||||
"twitter:account_id": "5536782",
|
||||
"og:type": "article",
|
||||
"twitter:title": "Sheikh Hasina forced to resign: What happened and what’s next?",
|
||||
"taxonomy-tags": "News, Sheikh Hasina",
|
||||
"topics": "News",
|
||||
"og:title": "Sheikh Hasina forced to resign: What happened and what’s next?",
|
||||
"tags": "Sheikh Hasina",
|
||||
"contenttype": "post",
|
||||
"twitter:image:src": "https://www.aljazeera.com/wp-content/uploads/2024/08/AP24218390076912-1722855595.jpg?resize=1920%2C1440",
|
||||
"articleslug": "bangladeshs-sheikh-hasina-forced-to-resign-what-happened-and-whats-next",
|
||||
"postlink": "/news/2024/8/5/bangladeshs-sheikh-hasina-forced-to-resign-what-happened-and-whats-next",
|
||||
"viewport": "width=device-width,initial-scale=1,shrink-to-fit=no",
|
||||
"twitter:description": "Prime minister reportedly flees to India after weeks of antigovernment protests.",
|
||||
"pagetitle": "Bangladesh’s Sheikh Hasina forced to resign: What happened and what’s next?",
|
||||
"og:url": "https://www.aljazeera.com/news/2024/8/5/bangladeshs-sheikh-hasina-forced-to-resign-what-happened-and-whats-next"
|
||||
}
|
||||
],
|
||||
"cse_image": [
|
||||
{
|
||||
"src": "https://www.aljazeera.com/wp-content/uploads/2024/08/AP24218390076912-1722855595.jpg?resize=1920%2C1440"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
28
search_response_altnews_in.html
Normal file
28
search_response_altnews_in.html
Normal file
File diff suppressed because one or more lines are too long
28
search_response_bbc_com.html
Normal file
28
search_response_bbc_com.html
Normal file
File diff suppressed because one or more lines are too long
28
search_response_en_prothomalo_com.html
Normal file
28
search_response_en_prothomalo_com.html
Normal file
File diff suppressed because one or more lines are too long
Loading…
Add table
Reference in a new issue