Graphics 800,600,16,2

outputfile$ = "sites\sites0.txt"

Global file = WriteFile(outputfile$)

Type url2do
	Field url$
	Field server$
End Type


readurls("blitzforum.de/","blitzforum.de")

For url_new.url2do = Each url2do
	DebugLog url_new.url2do\url+" "+ url_new.url2do\server
	readurls(url_new.url2do\url, url_new.url2do\server)
	url_done.url2do = First url2do
	Delete url_done.url2do
	
	If GetKey() Then 
		If Input("Wirklich beenden?")="JA" Then 
			Exit
		EndIf
	EndIf
	
	If FileSize(outputfile$)=>1048576 Then ;Wenn Datei grösser als 1048576 Bytes (1MB) dann neue Datei erstellen
		count = count + 1
		outputfile$ = "sites\sites"+Str(count)+".txt"
		CloseFile file
		file = WriteFile(outputfile$)
	EndIf
Next

CloseFile file

Function readurls(starturl$,server$)
stream = OpenTCPStream(server$,80)
If Not stream Then Return
WriteLine stream,"GET http://"+starturl$+" HTTP/1.0 USER-AGENT: Spindexer1.0"
WriteLine stream,Chr$(10)
While Not Eof(stream)
	zeile$ = ReadLine$(stream)
	;Print zeile$
	If Instr(zeile$,"<a href=") Then
		If Left(ParseContent(zeile$,"href"),1)="/" And  Left(ParseContent(zeile$,"href"),7) <> "HTTP://" And Left(ParseContent(zeile$,"href"),10) <> "JAVASCRIPT" And Left(ParseContent(zeile$,"href"),7) <> "MAILTO:" Then
			WriteLine file, starturl$+ParseContent(zeile$,"href")
			url_new.url2do = New url2do
			url_new.url2do\url = starturl$+ParseContent(zeile$,"href")
			url_new.url2do\server = server$
		ElseIf Left(ParseContent(zeile$,"href"),7) <> "HTTP://" And Left(ParseContent(zeile$,"href"),10) <> "JAVASCRIPT" And Left(ParseContent(zeile$,"href"),7) <> "MAILTO:"
			WriteLine file, starturl$+ParseContent(zeile$,"href")
			url_new.url2do = New url2do
			url_new.url2do\url = starturl$+ParseContent(zeile$,"href")
			url_new.url2do\server = server$
		ElseIf Left(ParseContent(zeile$,"href"),10) <> "JAVASCRIPT" And Left(ParseContent(zeile$,"href"),7) <> "MAILTO:"
			WriteLine file, ParseContent(zeile$,"href")
			url_new.url2do = New url2do
			url_new.url2do\url = ParseContent(zeile$,"href")
			temp$ = Replace(ParseContent(zeile$,"href"),"HTTP://","")
			If Instr(temp$,"/") > 0 Then
				url_new.url2do\server = Left(temp$,Instr(temp$,"/")-1)
			Else
				url_new.url2do\server = temp$
			EndIf
		EndIf
		;Print zeile$
	EndIf
Wend
CloseTCPStream stream
End Function


Function ParseContent$(inh$,tag$,ext$="=")
 inh$=Upper(inh$)
 par$=Upper(tag$+ext$)
 If Instr(inh$,par$) Then
  b=Instr(inh$," ",Instr(inh$,par$)+1)-Instr(inh$,par$)
  a=Instr(inh$,">",Instr(inh$,par$)+1)-Instr(inh$,par$)
  If a>b And b>0 Then
   sizepart$=Mid$(inh$,Instr(inh$,par$),b)
  ElseIf a>0 Then 
   sizepart$=Mid$(inh$,Instr(inh$,par$),a)
  End If
  change$=Replace$(Mid$(sizepart$,Instr(sizepart$,"=")+1,Len(sizepart$)-Instr(sizepart$,"=")+1),Chr$(34),"")
  Return change$
 End If
End Function