<?php
/**
* 爬取 国外高校犯罪信息数据
* Class SchoolSpider
*/
class SchoolSpider
{
private $schoolName;
//学校名检查api
private $schoolApi = 'https://ope.ed.gov/campussafety/api/institution/names?filter=';
//校区
private $schoolSerachApi = 'https://ope.ed.gov/campussafety/api/institution/search';
//数据 校区
private $dataApi = 'https://ope.ed.gov/campussafety/api/campus/';
//整个学校
private $totalApi = 'https://ope.ed.gov/campussafety/api/institution/';
//校区
private $campisesData;
//详情数据
private $dataDetail;
//详情数据
public $error;
public $result=[];
public function __construct()
{
//请求时间不限
set_time_limit(0);
}
public function spiderData($schoolName)
{
$this->schoolName = $schoolName;
$this->result['school_name'] = $this->schoolName;
if (!$this->checkName() || !$this->getCampises()) {
return ['code' => -1, 'msg' => $this->error];
}
//总的数据
$this->formatData($this->campisesData);
//各个分校区数据
$this->formatData($this->campisesData[0]['Campuses'],1);
return ['code' => 0, 'msg' => $this->error,'data'=>$this->result];
}
/**
* @name 数据解析
* @param $campisesData 数据
* @param $isCampise 是否是分校区数据
*/
private function formatData($campisesData,$isCampise=false){
foreach ($campisesData as $campise) {
if($isCampise){
//校区名
$res = $this->getDetailCampuse($campise['UnitID']);
}else{
$res = $this->getDetail($campise['UnitID']);
}
if(!$res||isset($this->result[$campise['UnitID']])){
continue;
}
$dataDetail = $this->dataDetail;
//校名
$this->result[$campise['UnitID']]['campuse'] = $campise['Name'];
foreach ($dataDetail as $k => $data) {
if ($k == 0)
continue;
$this->result[$campise['UnitID']]['data'][] = empty($data) ? [] : array_column($data['Cells'], 'Html');
}
}
}
/**
* 检查学校名
* @return array
*/
private function checkName()
{
$school = json_decode($this->httpRequest($this->schoolApi . urlencode($this->schoolName)), true);
if (!isset($school[0]['Code']) || !$school[0]['Code']) {
$this->error = '检查学校名 error';
return false;
}
return true;
}
/**
* 获取校区
* @return array
*/
private function getCampises()
{
$schoolSerachData = "{\"name\":\"" . $this->schoolName . "\",\"city\":\"\",\"state\":[],\"country\":[],\"countryNames\":[],\"institutionType\":[],\"institutionProgram\":[],\"campusLocation\":\"-1\",\"onlyResidentialCampuses\":false,\"enrollmentRange\":[],\"sort\":\"name\",\"sortDirection\":\"asc\",\"all\":false,\"pageNumber\":0,\"fromFavorites\":false}";
//获取校区列表
$campisesData = json_decode($this->httpRequest($this->schoolSerachApi, $schoolSerachData), true);
if (!isset($campisesData['Results'][0]['Campuses']) || empty($campisesData['Results'][0]['Campuses'])) {
$this->error = '获取校区 error';
return false;
}
$this->campisesData = $campisesData['Results'];
return true;
}
/**
* @name 获取全部详情
* @param $unitId
* @return array
*/
private function getDetail($unitId)
{
$dataDetail = json_decode($this->httpRequest($this->totalApi . $unitId), true);
if (!isset($dataDetail['Groups'][0]['Screens'][0]['Rows']) || empty($dataDetail['Groups'][0]['Screens'][0]['Rows'])) {
$this->error = '获取详情 error';
return false;
}
$this->dataDetail = $dataDetail['Groups'][0]['Screens'][0]['Rows'];
return true;
}
/**
* @name 获取分校区详情
* @param $unitId
* @return array
*/
private function getDetailCampuse($unitId)
{
$dataDetail = json_decode($this->httpRequest($this->dataApi . $unitId), true);
if (!isset($dataDetail['Groups'][0]['Screens'][0]['Rows']) || empty($dataDetail['Groups'][0]['Screens'][0]['Rows'])) {
$this->error = '获取详情 error';
return false;
}
$this->dataDetail = $dataDetail['Groups'][0]['Screens'][0]['Rows'];
return true;
}
/**
* @name 请求方法
* @param $url
* @param $data
* @return false|mixed|string
*/
private function httpRequest($url, $data = '')
{
$curl = curl_init();
$method = $data ? 'POST' : 'GET';
curl_setopt_array($curl, array(
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => $method,
CURLOPT_POSTFIELDS => $data,
CURLOPT_HTTPHEADER => array(
"Content-Type: application/json;charset=UTF-8",
"Postman-Token: a30f51b0-cc09-4bfb-801e-830a931994f4",
"cache-control: no-cache"
),
));
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
return json_encode([]);
} else {
return strip_tags($response);
}
}
}
$a = new SchoolSpider();
$r = $a->spiderData('Harvard University');
echo json_encode($r);
